@deepagents/text2sql 0.14.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +0 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2012 -3287
- package/dist/index.js.map +4 -4
- package/dist/lib/adapters/adapter.d.ts +2 -0
- package/dist/lib/adapters/adapter.d.ts.map +1 -1
- package/dist/lib/adapters/bigquery/bigquery.d.ts +41 -0
- package/dist/lib/adapters/bigquery/bigquery.d.ts.map +1 -0
- package/dist/lib/adapters/bigquery/constraint.bigquery.grounding.d.ts +11 -0
- package/dist/lib/adapters/bigquery/constraint.bigquery.grounding.d.ts.map +1 -0
- package/dist/lib/adapters/bigquery/index.d.ts +35 -0
- package/dist/lib/adapters/bigquery/index.d.ts.map +1 -0
- package/dist/lib/adapters/bigquery/index.js +1321 -0
- package/dist/lib/adapters/bigquery/index.js.map +7 -0
- package/dist/lib/adapters/bigquery/indexes.bigquery.grounding.d.ts +14 -0
- package/dist/lib/adapters/bigquery/indexes.bigquery.grounding.d.ts.map +1 -0
- package/dist/lib/adapters/bigquery/info.bigquery.grounding.d.ts +9 -0
- package/dist/lib/adapters/bigquery/info.bigquery.grounding.d.ts.map +1 -0
- package/dist/lib/adapters/bigquery/row-count.bigquery.grounding.d.ts +14 -0
- package/dist/lib/adapters/bigquery/row-count.bigquery.grounding.d.ts.map +1 -0
- package/dist/lib/adapters/bigquery/table.bigquery.grounding.d.ts +15 -0
- package/dist/lib/adapters/bigquery/table.bigquery.grounding.d.ts.map +1 -0
- package/dist/lib/adapters/bigquery/view.bigquery.grounding.d.ts +12 -0
- package/dist/lib/adapters/bigquery/view.bigquery.grounding.d.ts.map +1 -0
- package/dist/lib/adapters/groundings/index.js +11 -2058
- package/dist/lib/adapters/groundings/index.js.map +4 -4
- package/dist/lib/adapters/mysql/index.js +11 -2058
- package/dist/lib/adapters/mysql/index.js.map +4 -4
- package/dist/lib/adapters/postgres/column-stats.postgres.grounding.d.ts +2 -4
- package/dist/lib/adapters/postgres/column-stats.postgres.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/postgres/column-values.postgres.grounding.d.ts +0 -8
- package/dist/lib/adapters/postgres/column-values.postgres.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/postgres/index.js +139 -2068
- package/dist/lib/adapters/postgres/index.js.map +4 -4
- package/dist/lib/adapters/postgres/row-count.postgres.grounding.d.ts +0 -3
- package/dist/lib/adapters/postgres/row-count.postgres.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/spreadsheet/index.js +11 -56
- package/dist/lib/adapters/spreadsheet/index.js.map +4 -4
- package/dist/lib/adapters/sqlite/index.js +11 -2058
- package/dist/lib/adapters/sqlite/index.js.map +4 -4
- package/dist/lib/adapters/sqlserver/index.js +11 -2058
- package/dist/lib/adapters/sqlserver/index.js.map +4 -4
- package/dist/lib/agents/result-tools.d.ts +1 -3
- package/dist/lib/agents/result-tools.d.ts.map +1 -1
- package/dist/lib/fragments/schema.d.ts +2 -0
- package/dist/lib/fragments/schema.d.ts.map +1 -1
- package/dist/lib/fs/index.d.ts +2 -1
- package/dist/lib/fs/index.d.ts.map +1 -1
- package/dist/lib/fs/mssql/ddl.mssql-fs.d.ts +2 -0
- package/dist/lib/fs/mssql/ddl.mssql-fs.d.ts.map +1 -0
- package/dist/lib/fs/mssql/mssql-fs.d.ts +56 -0
- package/dist/lib/fs/mssql/mssql-fs.d.ts.map +1 -0
- package/dist/lib/fs/scoped-fs.d.ts +2 -0
- package/dist/lib/fs/scoped-fs.d.ts.map +1 -1
- package/dist/lib/fs/{sqlite-fs.d.ts → sqlite/sqlite-fs.d.ts} +2 -0
- package/dist/lib/fs/sqlite/sqlite-fs.d.ts.map +1 -0
- package/dist/lib/fs/tracked-fs.d.ts +2 -0
- package/dist/lib/fs/tracked-fs.d.ts.map +1 -1
- package/dist/lib/instructions.d.ts.map +1 -1
- package/dist/lib/sql.d.ts +2 -2
- package/dist/lib/sql.d.ts.map +1 -1
- package/dist/lib/synthesis/index.js +254 -2119
- package/dist/lib/synthesis/index.js.map +4 -4
- package/package.json +17 -10
- package/dist/lib/agents/text2sql.agent.d.ts +0 -3
- package/dist/lib/agents/text2sql.agent.d.ts.map +0 -1
- package/dist/lib/fs/sqlite-fs.d.ts.map +0 -1
|
@@ -165,1968 +165,22 @@ import {
|
|
|
165
165
|
} from "ai";
|
|
166
166
|
|
|
167
167
|
// packages/text2sql/src/lib/synthesis/extractors/base-contextual-extractor.ts
|
|
168
|
-
import { groq
|
|
169
|
-
import {
|
|
170
|
-
getToolOrDynamicToolName,
|
|
171
|
-
isTextUIPart,
|
|
172
|
-
isToolOrDynamicToolUIPart
|
|
173
|
-
} from "ai";
|
|
174
|
-
import
|
|
175
|
-
import z from "zod";
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
import { defineCommand } from "just-bash";
|
|
185
|
-
import spawn from "nano-spawn";
|
|
186
|
-
import "bash-tool";
|
|
187
|
-
import spawn2 from "nano-spawn";
|
|
188
|
-
import {
|
|
189
|
-
createBashTool
|
|
190
|
-
} from "bash-tool";
|
|
191
|
-
import dedent from "dedent";
|
|
192
|
-
import YAML from "yaml";
|
|
193
|
-
import { DatabaseSync } from "node:sqlite";
|
|
194
|
-
import { groq } from "@ai-sdk/groq";
|
|
195
|
-
import {
|
|
196
|
-
NoSuchToolError,
|
|
197
|
-
Output,
|
|
198
|
-
convertToModelMessages,
|
|
199
|
-
createUIMessageStream,
|
|
200
|
-
generateId as generateId2,
|
|
201
|
-
generateText,
|
|
202
|
-
smoothStream,
|
|
203
|
-
stepCountIs,
|
|
204
|
-
streamText
|
|
205
|
-
} from "ai";
|
|
206
|
-
import chalk2 from "chalk";
|
|
207
|
-
import "zod";
|
|
208
|
-
import "@deepagents/agent";
|
|
209
|
-
var defaultTokenizer = {
|
|
210
|
-
encode(text) {
|
|
211
|
-
return encode(text);
|
|
212
|
-
},
|
|
213
|
-
count(text) {
|
|
214
|
-
return encode(text).length;
|
|
215
|
-
}
|
|
216
|
-
};
|
|
217
|
-
var ModelsRegistry = class {
|
|
218
|
-
#cache = /* @__PURE__ */ new Map();
|
|
219
|
-
#loaded = false;
|
|
220
|
-
#tokenizers = /* @__PURE__ */ new Map();
|
|
221
|
-
#defaultTokenizer = defaultTokenizer;
|
|
222
|
-
/**
|
|
223
|
-
* Load models data from models.dev API
|
|
224
|
-
*/
|
|
225
|
-
async load() {
|
|
226
|
-
if (this.#loaded) return;
|
|
227
|
-
const response = await fetch("https://models.dev/api.json");
|
|
228
|
-
if (!response.ok) {
|
|
229
|
-
throw new Error(`Failed to fetch models: ${response.statusText}`);
|
|
230
|
-
}
|
|
231
|
-
const data = await response.json();
|
|
232
|
-
for (const [providerId, provider] of Object.entries(data)) {
|
|
233
|
-
for (const [modelId, model] of Object.entries(provider.models)) {
|
|
234
|
-
const info = {
|
|
235
|
-
id: model.id,
|
|
236
|
-
name: model.name,
|
|
237
|
-
family: model.family,
|
|
238
|
-
cost: model.cost,
|
|
239
|
-
limit: model.limit,
|
|
240
|
-
provider: providerId
|
|
241
|
-
};
|
|
242
|
-
this.#cache.set(`${providerId}:${modelId}`, info);
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
|
-
this.#loaded = true;
|
|
246
|
-
}
|
|
247
|
-
/**
|
|
248
|
-
* Get model info by ID
|
|
249
|
-
* @param modelId - Model ID (e.g., "openai:gpt-4o")
|
|
250
|
-
*/
|
|
251
|
-
get(modelId) {
|
|
252
|
-
return this.#cache.get(modelId);
|
|
253
|
-
}
|
|
254
|
-
/**
|
|
255
|
-
* Check if a model exists in the registry
|
|
256
|
-
*/
|
|
257
|
-
has(modelId) {
|
|
258
|
-
return this.#cache.has(modelId);
|
|
259
|
-
}
|
|
260
|
-
/**
|
|
261
|
-
* List all available model IDs
|
|
262
|
-
*/
|
|
263
|
-
list() {
|
|
264
|
-
return [...this.#cache.keys()];
|
|
265
|
-
}
|
|
266
|
-
/**
|
|
267
|
-
* Register a custom tokenizer for specific model families
|
|
268
|
-
* @param family - Model family name (e.g., "llama", "claude")
|
|
269
|
-
* @param tokenizer - Tokenizer implementation
|
|
270
|
-
*/
|
|
271
|
-
registerTokenizer(family, tokenizer) {
|
|
272
|
-
this.#tokenizers.set(family, tokenizer);
|
|
273
|
-
}
|
|
274
|
-
/**
|
|
275
|
-
* Set the default tokenizer used when no family-specific tokenizer is registered
|
|
276
|
-
*/
|
|
277
|
-
setDefaultTokenizer(tokenizer) {
|
|
278
|
-
this.#defaultTokenizer = tokenizer;
|
|
279
|
-
}
|
|
280
|
-
/**
|
|
281
|
-
* Get the appropriate tokenizer for a model
|
|
282
|
-
*/
|
|
283
|
-
getTokenizer(modelId) {
|
|
284
|
-
const model = this.get(modelId);
|
|
285
|
-
if (model) {
|
|
286
|
-
const familyTokenizer = this.#tokenizers.get(model.family);
|
|
287
|
-
if (familyTokenizer) {
|
|
288
|
-
return familyTokenizer;
|
|
289
|
-
}
|
|
290
|
-
}
|
|
291
|
-
return this.#defaultTokenizer;
|
|
292
|
-
}
|
|
293
|
-
/**
|
|
294
|
-
* Estimate token count and cost for given text and model
|
|
295
|
-
* @param modelId - Model ID to use for pricing (e.g., "openai:gpt-4o")
|
|
296
|
-
* @param input - Input text (prompt)
|
|
297
|
-
*/
|
|
298
|
-
estimate(modelId, input) {
|
|
299
|
-
const model = this.get(modelId);
|
|
300
|
-
if (!model) {
|
|
301
|
-
throw new Error(
|
|
302
|
-
`Model "${modelId}" not found. Call load() first or check model ID.`
|
|
303
|
-
);
|
|
304
|
-
}
|
|
305
|
-
const tokenizer = this.getTokenizer(modelId);
|
|
306
|
-
const tokens = tokenizer.count(input);
|
|
307
|
-
const cost = tokens / 1e6 * model.cost.input;
|
|
308
|
-
return {
|
|
309
|
-
model: model.id,
|
|
310
|
-
provider: model.provider,
|
|
311
|
-
tokens,
|
|
312
|
-
cost,
|
|
313
|
-
limits: {
|
|
314
|
-
context: model.limit.context,
|
|
315
|
-
output: model.limit.output,
|
|
316
|
-
exceedsContext: tokens > model.limit.context
|
|
317
|
-
},
|
|
318
|
-
fragments: []
|
|
319
|
-
};
|
|
320
|
-
}
|
|
321
|
-
};
|
|
322
|
-
var _registry = null;
|
|
323
|
-
function getModelsRegistry() {
|
|
324
|
-
if (!_registry) {
|
|
325
|
-
_registry = new ModelsRegistry();
|
|
326
|
-
}
|
|
327
|
-
return _registry;
|
|
328
|
-
}
|
|
329
|
-
function isFragment(data) {
|
|
330
|
-
return typeof data === "object" && data !== null && "name" in data && "data" in data && typeof data.name === "string";
|
|
331
|
-
}
|
|
332
|
-
function isFragmentObject(data) {
|
|
333
|
-
return typeof data === "object" && data !== null && !Array.isArray(data) && !isFragment(data);
|
|
334
|
-
}
|
|
335
|
-
function isMessageFragment(fragment2) {
|
|
336
|
-
return fragment2.type === "message";
|
|
337
|
-
}
|
|
338
|
-
function fragment(name, ...children) {
|
|
339
|
-
return {
|
|
340
|
-
name,
|
|
341
|
-
data: children
|
|
342
|
-
};
|
|
343
|
-
}
|
|
344
|
-
function user(content) {
|
|
345
|
-
const message2 = typeof content === "string" ? {
|
|
346
|
-
id: generateId(),
|
|
347
|
-
role: "user",
|
|
348
|
-
parts: [{ type: "text", text: content }]
|
|
349
|
-
} : content;
|
|
350
|
-
return {
|
|
351
|
-
id: message2.id,
|
|
352
|
-
name: "user",
|
|
353
|
-
data: "content",
|
|
354
|
-
type: "message",
|
|
355
|
-
persist: true,
|
|
356
|
-
codec: {
|
|
357
|
-
decode() {
|
|
358
|
-
return message2;
|
|
359
|
-
},
|
|
360
|
-
encode() {
|
|
361
|
-
return message2;
|
|
362
|
-
}
|
|
363
|
-
}
|
|
364
|
-
};
|
|
365
|
-
}
|
|
366
|
-
function assistant(message2) {
|
|
367
|
-
return {
|
|
368
|
-
id: message2.id,
|
|
369
|
-
name: "assistant",
|
|
370
|
-
data: "content",
|
|
371
|
-
type: "message",
|
|
372
|
-
persist: true,
|
|
373
|
-
codec: {
|
|
374
|
-
decode() {
|
|
375
|
-
return message2;
|
|
376
|
-
},
|
|
377
|
-
encode() {
|
|
378
|
-
return message2;
|
|
379
|
-
}
|
|
380
|
-
}
|
|
381
|
-
};
|
|
382
|
-
}
|
|
383
|
-
function message(content) {
|
|
384
|
-
const message2 = typeof content === "string" ? {
|
|
385
|
-
id: generateId(),
|
|
386
|
-
role: "user",
|
|
387
|
-
parts: [{ type: "text", text: content }]
|
|
388
|
-
} : content;
|
|
389
|
-
return {
|
|
390
|
-
id: message2.id,
|
|
391
|
-
name: message2.role,
|
|
392
|
-
data: "content",
|
|
393
|
-
type: "message",
|
|
394
|
-
persist: true,
|
|
395
|
-
codec: {
|
|
396
|
-
decode() {
|
|
397
|
-
return message2;
|
|
398
|
-
},
|
|
399
|
-
encode() {
|
|
400
|
-
return message2;
|
|
401
|
-
}
|
|
402
|
-
}
|
|
403
|
-
};
|
|
404
|
-
}
|
|
405
|
-
function assistantText(content, options) {
|
|
406
|
-
const id = options?.id ?? crypto.randomUUID();
|
|
407
|
-
return assistant({
|
|
408
|
-
id,
|
|
409
|
-
role: "assistant",
|
|
410
|
-
parts: [{ type: "text", text: content }]
|
|
411
|
-
});
|
|
412
|
-
}
|
|
413
|
-
var LAZY_ID = Symbol.for("@deepagents/context:lazy-id");
|
|
414
|
-
function isLazyFragment(fragment2) {
|
|
415
|
-
return LAZY_ID in fragment2;
|
|
416
|
-
}
|
|
417
|
-
var ContextRenderer = class {
|
|
418
|
-
options;
|
|
419
|
-
constructor(options = {}) {
|
|
420
|
-
this.options = options;
|
|
421
|
-
}
|
|
422
|
-
/**
|
|
423
|
-
* Check if data is a primitive (string, number, boolean).
|
|
424
|
-
*/
|
|
425
|
-
isPrimitive(data) {
|
|
426
|
-
return typeof data === "string" || typeof data === "number" || typeof data === "boolean";
|
|
427
|
-
}
|
|
428
|
-
/**
|
|
429
|
-
* Group fragments by name for groupFragments option.
|
|
430
|
-
*/
|
|
431
|
-
groupByName(fragments) {
|
|
432
|
-
const groups = /* @__PURE__ */ new Map();
|
|
433
|
-
for (const fragment2 of fragments) {
|
|
434
|
-
const existing = groups.get(fragment2.name) ?? [];
|
|
435
|
-
existing.push(fragment2);
|
|
436
|
-
groups.set(fragment2.name, existing);
|
|
437
|
-
}
|
|
438
|
-
return groups;
|
|
439
|
-
}
|
|
440
|
-
/**
|
|
441
|
-
* Remove null/undefined from fragments and fragment data recursively.
|
|
442
|
-
* This protects renderers from nullish values and ensures they are ignored
|
|
443
|
-
* consistently across all output formats.
|
|
444
|
-
*/
|
|
445
|
-
sanitizeFragments(fragments) {
|
|
446
|
-
const sanitized = [];
|
|
447
|
-
for (const fragment2 of fragments) {
|
|
448
|
-
const cleaned = this.sanitizeFragment(fragment2, /* @__PURE__ */ new WeakSet());
|
|
449
|
-
if (cleaned) {
|
|
450
|
-
sanitized.push(cleaned);
|
|
451
|
-
}
|
|
452
|
-
}
|
|
453
|
-
return sanitized;
|
|
454
|
-
}
|
|
455
|
-
sanitizeFragment(fragment2, seen) {
|
|
456
|
-
const data = this.sanitizeData(fragment2.data, seen);
|
|
457
|
-
if (data == null) {
|
|
458
|
-
return null;
|
|
459
|
-
}
|
|
460
|
-
return {
|
|
461
|
-
...fragment2,
|
|
462
|
-
data
|
|
463
|
-
};
|
|
464
|
-
}
|
|
465
|
-
sanitizeData(data, seen) {
|
|
466
|
-
if (data == null) {
|
|
467
|
-
return void 0;
|
|
468
|
-
}
|
|
469
|
-
if (isFragment(data)) {
|
|
470
|
-
return this.sanitizeFragment(data, seen) ?? void 0;
|
|
471
|
-
}
|
|
472
|
-
if (Array.isArray(data)) {
|
|
473
|
-
if (seen.has(data)) {
|
|
474
|
-
return void 0;
|
|
475
|
-
}
|
|
476
|
-
seen.add(data);
|
|
477
|
-
const cleaned = [];
|
|
478
|
-
for (const item of data) {
|
|
479
|
-
const sanitizedItem = this.sanitizeData(item, seen);
|
|
480
|
-
if (sanitizedItem != null) {
|
|
481
|
-
cleaned.push(sanitizedItem);
|
|
482
|
-
}
|
|
483
|
-
}
|
|
484
|
-
return cleaned;
|
|
485
|
-
}
|
|
486
|
-
if (isFragmentObject(data)) {
|
|
487
|
-
if (seen.has(data)) {
|
|
488
|
-
return void 0;
|
|
489
|
-
}
|
|
490
|
-
seen.add(data);
|
|
491
|
-
const cleaned = {};
|
|
492
|
-
for (const [key, value] of Object.entries(data)) {
|
|
493
|
-
const sanitizedValue = this.sanitizeData(value, seen);
|
|
494
|
-
if (sanitizedValue != null) {
|
|
495
|
-
cleaned[key] = sanitizedValue;
|
|
496
|
-
}
|
|
497
|
-
}
|
|
498
|
-
return cleaned;
|
|
499
|
-
}
|
|
500
|
-
return data;
|
|
501
|
-
}
|
|
502
|
-
/**
|
|
503
|
-
* Template method - dispatches value to appropriate handler.
|
|
504
|
-
*/
|
|
505
|
-
renderValue(key, value, ctx) {
|
|
506
|
-
if (value == null) {
|
|
507
|
-
return "";
|
|
508
|
-
}
|
|
509
|
-
if (isFragment(value)) {
|
|
510
|
-
return this.renderFragment(value, ctx);
|
|
511
|
-
}
|
|
512
|
-
if (Array.isArray(value)) {
|
|
513
|
-
return this.renderArray(key, value, ctx);
|
|
514
|
-
}
|
|
515
|
-
if (isFragmentObject(value)) {
|
|
516
|
-
return this.renderObject(key, value, ctx);
|
|
517
|
-
}
|
|
518
|
-
return this.renderPrimitive(key, String(value), ctx);
|
|
519
|
-
}
|
|
520
|
-
/**
|
|
521
|
-
* Render all entries of an object.
|
|
522
|
-
*/
|
|
523
|
-
renderEntries(data, ctx) {
|
|
524
|
-
return Object.entries(data).map(([key, value]) => this.renderValue(key, value, ctx)).filter(Boolean);
|
|
525
|
-
}
|
|
526
|
-
};
|
|
527
|
-
var XmlRenderer = class extends ContextRenderer {
|
|
528
|
-
render(fragments) {
|
|
529
|
-
const sanitized = this.sanitizeFragments(fragments);
|
|
530
|
-
return sanitized.map((f) => this.#renderTopLevel(f)).filter(Boolean).join("\n");
|
|
531
|
-
}
|
|
532
|
-
#renderTopLevel(fragment2) {
|
|
533
|
-
if (this.isPrimitive(fragment2.data)) {
|
|
534
|
-
return this.#leafRoot(fragment2.name, String(fragment2.data));
|
|
535
|
-
}
|
|
536
|
-
if (Array.isArray(fragment2.data)) {
|
|
537
|
-
return this.#renderArray(fragment2.name, fragment2.data, 0);
|
|
538
|
-
}
|
|
539
|
-
if (isFragment(fragment2.data)) {
|
|
540
|
-
const child = this.renderFragment(fragment2.data, { depth: 1, path: [] });
|
|
541
|
-
return this.#wrap(fragment2.name, [child]);
|
|
542
|
-
}
|
|
543
|
-
if (isFragmentObject(fragment2.data)) {
|
|
544
|
-
return this.#wrap(
|
|
545
|
-
fragment2.name,
|
|
546
|
-
this.renderEntries(fragment2.data, { depth: 1, path: [] })
|
|
547
|
-
);
|
|
548
|
-
}
|
|
549
|
-
return "";
|
|
550
|
-
}
|
|
551
|
-
#renderArray(name, items, depth) {
|
|
552
|
-
const fragmentItems = items.filter(isFragment);
|
|
553
|
-
const nonFragmentItems = items.filter((item) => !isFragment(item));
|
|
554
|
-
const children = [];
|
|
555
|
-
for (const item of nonFragmentItems) {
|
|
556
|
-
if (item != null) {
|
|
557
|
-
if (isFragmentObject(item)) {
|
|
558
|
-
children.push(
|
|
559
|
-
this.#wrapIndented(
|
|
560
|
-
pluralize.singular(name),
|
|
561
|
-
this.renderEntries(item, { depth: depth + 2, path: [] }),
|
|
562
|
-
depth + 1
|
|
563
|
-
)
|
|
564
|
-
);
|
|
565
|
-
} else {
|
|
566
|
-
children.push(
|
|
567
|
-
this.#leaf(pluralize.singular(name), String(item), depth + 1)
|
|
568
|
-
);
|
|
569
|
-
}
|
|
570
|
-
}
|
|
571
|
-
}
|
|
572
|
-
if (this.options.groupFragments && fragmentItems.length > 0) {
|
|
573
|
-
const groups = this.groupByName(fragmentItems);
|
|
574
|
-
for (const [groupName, groupFragments] of groups) {
|
|
575
|
-
const groupChildren = groupFragments.map(
|
|
576
|
-
(frag) => this.renderFragment(frag, { depth: depth + 2, path: [] })
|
|
577
|
-
);
|
|
578
|
-
const pluralName = pluralize.plural(groupName);
|
|
579
|
-
children.push(this.#wrapIndented(pluralName, groupChildren, depth + 1));
|
|
580
|
-
}
|
|
581
|
-
} else {
|
|
582
|
-
for (const frag of fragmentItems) {
|
|
583
|
-
children.push(
|
|
584
|
-
this.renderFragment(frag, { depth: depth + 1, path: [] })
|
|
585
|
-
);
|
|
586
|
-
}
|
|
587
|
-
}
|
|
588
|
-
return this.#wrap(name, children);
|
|
589
|
-
}
|
|
590
|
-
#leafRoot(tag, value) {
|
|
591
|
-
const safe = this.#escape(value);
|
|
592
|
-
if (safe.includes("\n")) {
|
|
593
|
-
return `<${tag}>
|
|
594
|
-
${this.#indent(safe, 2)}
|
|
595
|
-
</${tag}>`;
|
|
596
|
-
}
|
|
597
|
-
return `<${tag}>${safe}</${tag}>`;
|
|
598
|
-
}
|
|
599
|
-
renderFragment(fragment2, ctx) {
|
|
600
|
-
const { name, data } = fragment2;
|
|
601
|
-
if (this.isPrimitive(data)) {
|
|
602
|
-
return this.#leaf(name, String(data), ctx.depth);
|
|
603
|
-
}
|
|
604
|
-
if (isFragment(data)) {
|
|
605
|
-
const child = this.renderFragment(data, { ...ctx, depth: ctx.depth + 1 });
|
|
606
|
-
return this.#wrapIndented(name, [child], ctx.depth);
|
|
607
|
-
}
|
|
608
|
-
if (Array.isArray(data)) {
|
|
609
|
-
return this.#renderArrayIndented(name, data, ctx.depth);
|
|
610
|
-
}
|
|
611
|
-
if (isFragmentObject(data)) {
|
|
612
|
-
const children = this.renderEntries(data, {
|
|
613
|
-
...ctx,
|
|
614
|
-
depth: ctx.depth + 1
|
|
615
|
-
});
|
|
616
|
-
return this.#wrapIndented(name, children, ctx.depth);
|
|
617
|
-
}
|
|
618
|
-
return "";
|
|
619
|
-
}
|
|
620
|
-
#renderArrayIndented(name, items, depth) {
|
|
621
|
-
const fragmentItems = items.filter(isFragment);
|
|
622
|
-
const nonFragmentItems = items.filter((item) => !isFragment(item));
|
|
623
|
-
const children = [];
|
|
624
|
-
for (const item of nonFragmentItems) {
|
|
625
|
-
if (item != null) {
|
|
626
|
-
if (isFragmentObject(item)) {
|
|
627
|
-
children.push(
|
|
628
|
-
this.#wrapIndented(
|
|
629
|
-
pluralize.singular(name),
|
|
630
|
-
this.renderEntries(item, { depth: depth + 2, path: [] }),
|
|
631
|
-
depth + 1
|
|
632
|
-
)
|
|
633
|
-
);
|
|
634
|
-
} else {
|
|
635
|
-
children.push(
|
|
636
|
-
this.#leaf(pluralize.singular(name), String(item), depth + 1)
|
|
637
|
-
);
|
|
638
|
-
}
|
|
639
|
-
}
|
|
640
|
-
}
|
|
641
|
-
if (this.options.groupFragments && fragmentItems.length > 0) {
|
|
642
|
-
const groups = this.groupByName(fragmentItems);
|
|
643
|
-
for (const [groupName, groupFragments] of groups) {
|
|
644
|
-
const groupChildren = groupFragments.map(
|
|
645
|
-
(frag) => this.renderFragment(frag, { depth: depth + 2, path: [] })
|
|
646
|
-
);
|
|
647
|
-
const pluralName = pluralize.plural(groupName);
|
|
648
|
-
children.push(this.#wrapIndented(pluralName, groupChildren, depth + 1));
|
|
649
|
-
}
|
|
650
|
-
} else {
|
|
651
|
-
for (const frag of fragmentItems) {
|
|
652
|
-
children.push(
|
|
653
|
-
this.renderFragment(frag, { depth: depth + 1, path: [] })
|
|
654
|
-
);
|
|
655
|
-
}
|
|
656
|
-
}
|
|
657
|
-
return this.#wrapIndented(name, children, depth);
|
|
658
|
-
}
|
|
659
|
-
renderPrimitive(key, value, ctx) {
|
|
660
|
-
return this.#leaf(key, value, ctx.depth);
|
|
661
|
-
}
|
|
662
|
-
renderArray(key, items, ctx) {
|
|
663
|
-
if (!items.length) {
|
|
664
|
-
return "";
|
|
665
|
-
}
|
|
666
|
-
const itemTag = pluralize.singular(key);
|
|
667
|
-
const children = items.filter((item) => item != null).map((item) => {
|
|
668
|
-
if (isFragment(item)) {
|
|
669
|
-
return this.renderFragment(item, { ...ctx, depth: ctx.depth + 1 });
|
|
670
|
-
}
|
|
671
|
-
if (isFragmentObject(item)) {
|
|
672
|
-
return this.#wrapIndented(
|
|
673
|
-
itemTag,
|
|
674
|
-
this.renderEntries(item, { ...ctx, depth: ctx.depth + 2 }),
|
|
675
|
-
ctx.depth + 1
|
|
676
|
-
);
|
|
677
|
-
}
|
|
678
|
-
return this.#leaf(itemTag, String(item), ctx.depth + 1);
|
|
679
|
-
});
|
|
680
|
-
return this.#wrapIndented(key, children, ctx.depth);
|
|
681
|
-
}
|
|
682
|
-
renderObject(key, obj, ctx) {
|
|
683
|
-
const children = this.renderEntries(obj, { ...ctx, depth: ctx.depth + 1 });
|
|
684
|
-
return this.#wrapIndented(key, children, ctx.depth);
|
|
685
|
-
}
|
|
686
|
-
#escape(value) {
|
|
687
|
-
if (value == null) {
|
|
688
|
-
return "";
|
|
689
|
-
}
|
|
690
|
-
return value.replaceAll(/&/g, "&").replaceAll(/</g, "<").replaceAll(/>/g, ">").replaceAll(/"/g, """).replaceAll(/'/g, "'");
|
|
691
|
-
}
|
|
692
|
-
#indent(text, spaces) {
|
|
693
|
-
if (!text.trim()) {
|
|
694
|
-
return "";
|
|
695
|
-
}
|
|
696
|
-
const padding = " ".repeat(spaces);
|
|
697
|
-
return text.split("\n").map((line) => line.length ? padding + line : padding).join("\n");
|
|
698
|
-
}
|
|
699
|
-
#leaf(tag, value, depth) {
|
|
700
|
-
const safe = this.#escape(value);
|
|
701
|
-
const pad = " ".repeat(depth);
|
|
702
|
-
if (safe.includes("\n")) {
|
|
703
|
-
return `${pad}<${tag}>
|
|
704
|
-
${this.#indent(safe, (depth + 1) * 2)}
|
|
705
|
-
${pad}</${tag}>`;
|
|
706
|
-
}
|
|
707
|
-
return `${pad}<${tag}>${safe}</${tag}>`;
|
|
708
|
-
}
|
|
709
|
-
#wrap(tag, children) {
|
|
710
|
-
const content = children.filter(Boolean).join("\n");
|
|
711
|
-
if (!content) {
|
|
712
|
-
return "";
|
|
713
|
-
}
|
|
714
|
-
return `<${tag}>
|
|
715
|
-
${content}
|
|
716
|
-
</${tag}>`;
|
|
717
|
-
}
|
|
718
|
-
#wrapIndented(tag, children, depth) {
|
|
719
|
-
const content = children.filter(Boolean).join("\n");
|
|
720
|
-
if (!content) {
|
|
721
|
-
return "";
|
|
722
|
-
}
|
|
723
|
-
const pad = " ".repeat(depth);
|
|
724
|
-
return `${pad}<${tag}>
|
|
725
|
-
${content}
|
|
726
|
-
${pad}</${tag}>`;
|
|
727
|
-
}
|
|
728
|
-
};
|
|
729
|
-
var ContextStore = class {
|
|
730
|
-
};
|
|
731
|
-
var ContextEngine = class {
|
|
732
|
-
/** Non-message fragments (role, hints, etc.) - not persisted in graph */
|
|
733
|
-
#fragments = [];
|
|
734
|
-
/** Pending message fragments to be added to graph */
|
|
735
|
-
#pendingMessages = [];
|
|
736
|
-
#store;
|
|
737
|
-
#chatId;
|
|
738
|
-
#userId;
|
|
739
|
-
#branchName;
|
|
740
|
-
#branch = null;
|
|
741
|
-
#chatData = null;
|
|
742
|
-
#initialized = false;
|
|
743
|
-
/** Initial metadata to merge on first initialization */
|
|
744
|
-
#initialMetadata;
|
|
745
|
-
constructor(options) {
|
|
746
|
-
if (!options.chatId) {
|
|
747
|
-
throw new Error("chatId is required");
|
|
748
|
-
}
|
|
749
|
-
if (!options.userId) {
|
|
750
|
-
throw new Error("userId is required");
|
|
751
|
-
}
|
|
752
|
-
this.#store = options.store;
|
|
753
|
-
this.#chatId = options.chatId;
|
|
754
|
-
this.#userId = options.userId;
|
|
755
|
-
this.#branchName = "main";
|
|
756
|
-
this.#initialMetadata = options.metadata;
|
|
757
|
-
}
|
|
758
|
-
/**
|
|
759
|
-
* Initialize the chat and branch if they don't exist.
|
|
760
|
-
*/
|
|
761
|
-
async #ensureInitialized() {
|
|
762
|
-
if (this.#initialized) {
|
|
763
|
-
return;
|
|
764
|
-
}
|
|
765
|
-
this.#chatData = await this.#store.upsertChat({
|
|
766
|
-
id: this.#chatId,
|
|
767
|
-
userId: this.#userId
|
|
768
|
-
});
|
|
769
|
-
if (this.#initialMetadata) {
|
|
770
|
-
this.#chatData = await this.#store.updateChat(this.#chatId, {
|
|
771
|
-
metadata: {
|
|
772
|
-
...this.#chatData.metadata,
|
|
773
|
-
...this.#initialMetadata
|
|
774
|
-
}
|
|
775
|
-
});
|
|
776
|
-
this.#initialMetadata = void 0;
|
|
777
|
-
}
|
|
778
|
-
this.#branch = await this.#store.getActiveBranch(this.#chatId);
|
|
779
|
-
this.#initialized = true;
|
|
780
|
-
}
|
|
781
|
-
/**
|
|
782
|
-
* Create a new branch from a specific message.
|
|
783
|
-
* Shared logic between rewind() and btw().
|
|
784
|
-
*/
|
|
785
|
-
async #createBranchFrom(messageId, switchTo) {
|
|
786
|
-
const branches = await this.#store.listBranches(this.#chatId);
|
|
787
|
-
const samePrefix = branches.filter(
|
|
788
|
-
(it) => it.name === this.#branchName || it.name.startsWith(`${this.#branchName}-v`)
|
|
789
|
-
);
|
|
790
|
-
const newBranchName = `${this.#branchName}-v${samePrefix.length + 1}`;
|
|
791
|
-
const newBranch = {
|
|
792
|
-
id: crypto.randomUUID(),
|
|
793
|
-
chatId: this.#chatId,
|
|
794
|
-
name: newBranchName,
|
|
795
|
-
headMessageId: messageId,
|
|
796
|
-
isActive: false,
|
|
797
|
-
createdAt: Date.now()
|
|
798
|
-
};
|
|
799
|
-
await this.#store.createBranch(newBranch);
|
|
800
|
-
if (switchTo) {
|
|
801
|
-
await this.#store.setActiveBranch(this.#chatId, newBranch.id);
|
|
802
|
-
this.#branch = { ...newBranch, isActive: true };
|
|
803
|
-
this.#branchName = newBranchName;
|
|
804
|
-
this.#pendingMessages = [];
|
|
805
|
-
}
|
|
806
|
-
const chain = await this.#store.getMessageChain(messageId);
|
|
807
|
-
return {
|
|
808
|
-
id: newBranch.id,
|
|
809
|
-
name: newBranch.name,
|
|
810
|
-
headMessageId: newBranch.headMessageId,
|
|
811
|
-
isActive: switchTo,
|
|
812
|
-
messageCount: chain.length,
|
|
813
|
-
createdAt: newBranch.createdAt
|
|
814
|
-
};
|
|
815
|
-
}
|
|
816
|
-
/**
|
|
817
|
-
* Rewind to a message without clearing pending messages.
|
|
818
|
-
* Used internally when saving an update to an existing message.
|
|
819
|
-
*/
|
|
820
|
-
async #rewindForUpdate(messageId) {
|
|
821
|
-
const pendingBackup = [...this.#pendingMessages];
|
|
822
|
-
await this.rewind(messageId);
|
|
823
|
-
this.#pendingMessages = pendingBackup;
|
|
824
|
-
}
|
|
825
|
-
/**
|
|
826
|
-
* Get the current chat ID.
|
|
827
|
-
*/
|
|
828
|
-
get chatId() {
|
|
829
|
-
return this.#chatId;
|
|
830
|
-
}
|
|
831
|
-
/**
|
|
832
|
-
* Get the current branch name.
|
|
833
|
-
*/
|
|
834
|
-
get branch() {
|
|
835
|
-
return this.#branchName;
|
|
836
|
-
}
|
|
837
|
-
/**
|
|
838
|
-
* Get metadata for the current chat.
|
|
839
|
-
* Returns null if the chat hasn't been initialized yet.
|
|
840
|
-
*/
|
|
841
|
-
get chat() {
|
|
842
|
-
if (!this.#chatData) {
|
|
843
|
-
return null;
|
|
844
|
-
}
|
|
845
|
-
return {
|
|
846
|
-
id: this.#chatData.id,
|
|
847
|
-
userId: this.#chatData.userId,
|
|
848
|
-
createdAt: this.#chatData.createdAt,
|
|
849
|
-
updatedAt: this.#chatData.updatedAt,
|
|
850
|
-
title: this.#chatData.title,
|
|
851
|
-
metadata: this.#chatData.metadata
|
|
852
|
-
};
|
|
853
|
-
}
|
|
854
|
-
/**
|
|
855
|
-
* Add fragments to the context.
|
|
856
|
-
*
|
|
857
|
-
* - Message fragments (user/assistant) are queued for persistence
|
|
858
|
-
* - Non-message fragments (role/hint) are kept in memory for system prompt
|
|
859
|
-
*/
|
|
860
|
-
set(...fragments) {
|
|
861
|
-
for (const fragment2 of fragments) {
|
|
862
|
-
if (isMessageFragment(fragment2)) {
|
|
863
|
-
this.#pendingMessages.push(fragment2);
|
|
864
|
-
} else {
|
|
865
|
-
this.#fragments.push(fragment2);
|
|
866
|
-
}
|
|
867
|
-
}
|
|
868
|
-
return this;
|
|
869
|
-
}
|
|
870
|
-
// Unset a fragment by ID (not implemented yet)
|
|
871
|
-
unset(fragmentId) {
|
|
872
|
-
}
|
|
873
|
-
/**
|
|
874
|
-
* Render all fragments using the provided renderer.
|
|
875
|
-
* @internal Use resolve() instead for public API.
|
|
876
|
-
*/
|
|
877
|
-
render(renderer) {
|
|
878
|
-
return renderer.render(this.#fragments);
|
|
879
|
-
}
|
|
880
|
-
/**
|
|
881
|
-
* Resolve context into AI SDK-ready format.
|
|
882
|
-
*
|
|
883
|
-
* - Initializes chat and branch if needed
|
|
884
|
-
* - Loads message history from the graph (walking parent chain)
|
|
885
|
-
* - Separates context fragments for system prompt
|
|
886
|
-
* - Combines with pending messages
|
|
887
|
-
*
|
|
888
|
-
* @example
|
|
889
|
-
* ```ts
|
|
890
|
-
* const context = new ContextEngine({ store, chatId: 'chat-1', userId: 'user-1' })
|
|
891
|
-
* .set(role('You are helpful'), user('Hello'));
|
|
892
|
-
*
|
|
893
|
-
* const { systemPrompt, messages } = await context.resolve();
|
|
894
|
-
* await generateText({ system: systemPrompt, messages });
|
|
895
|
-
* ```
|
|
896
|
-
*/
|
|
897
|
-
async resolve(options) {
|
|
898
|
-
await this.#ensureInitialized();
|
|
899
|
-
const systemPrompt = options.renderer.render(this.#fragments);
|
|
900
|
-
const messages = [];
|
|
901
|
-
if (this.#branch?.headMessageId) {
|
|
902
|
-
const chain = await this.#store.getMessageChain(
|
|
903
|
-
this.#branch.headMessageId
|
|
904
|
-
);
|
|
905
|
-
for (const msg of chain) {
|
|
906
|
-
messages.push(message(msg.data).codec?.decode());
|
|
907
|
-
}
|
|
908
|
-
}
|
|
909
|
-
for (let i = 0; i < this.#pendingMessages.length; i++) {
|
|
910
|
-
const fragment2 = this.#pendingMessages[i];
|
|
911
|
-
if (isLazyFragment(fragment2)) {
|
|
912
|
-
this.#pendingMessages[i] = await this.#resolveLazyFragment(fragment2);
|
|
913
|
-
}
|
|
914
|
-
}
|
|
915
|
-
for (const fragment2 of this.#pendingMessages) {
|
|
916
|
-
if (!fragment2.codec) {
|
|
917
|
-
throw new Error(
|
|
918
|
-
`Fragment "${fragment2.name}" is missing codec. Lazy fragments must be resolved before decode.`
|
|
919
|
-
);
|
|
920
|
-
}
|
|
921
|
-
const decoded = fragment2.codec.decode();
|
|
922
|
-
messages.push(decoded);
|
|
923
|
-
}
|
|
924
|
-
return { systemPrompt, messages };
|
|
925
|
-
}
|
|
926
|
-
/**
|
|
927
|
-
* Save pending messages to the graph.
|
|
928
|
-
*
|
|
929
|
-
* Each message is added as a node with parentId pointing to the previous message.
|
|
930
|
-
* The branch head is updated to point to the last message.
|
|
931
|
-
*
|
|
932
|
-
* @example
|
|
933
|
-
* ```ts
|
|
934
|
-
* context.set(user('Hello'));
|
|
935
|
-
* // AI responds...
|
|
936
|
-
* context.set(assistant('Hi there!'));
|
|
937
|
-
* await context.save(); // Persist to graph
|
|
938
|
-
* ```
|
|
939
|
-
*/
|
|
940
|
-
async save() {
|
|
941
|
-
await this.#ensureInitialized();
|
|
942
|
-
if (this.#pendingMessages.length === 0) {
|
|
943
|
-
return;
|
|
944
|
-
}
|
|
945
|
-
for (let i = 0; i < this.#pendingMessages.length; i++) {
|
|
946
|
-
const fragment2 = this.#pendingMessages[i];
|
|
947
|
-
if (isLazyFragment(fragment2)) {
|
|
948
|
-
this.#pendingMessages[i] = await this.#resolveLazyFragment(fragment2);
|
|
949
|
-
}
|
|
950
|
-
}
|
|
951
|
-
for (const fragment2 of this.#pendingMessages) {
|
|
952
|
-
if (fragment2.id) {
|
|
953
|
-
const existing = await this.#store.getMessage(fragment2.id);
|
|
954
|
-
if (existing && existing.parentId) {
|
|
955
|
-
await this.#rewindForUpdate(existing.parentId);
|
|
956
|
-
fragment2.id = crypto.randomUUID();
|
|
957
|
-
break;
|
|
958
|
-
}
|
|
959
|
-
}
|
|
960
|
-
}
|
|
961
|
-
let parentId = this.#branch.headMessageId;
|
|
962
|
-
const now = Date.now();
|
|
963
|
-
for (const fragment2 of this.#pendingMessages) {
|
|
964
|
-
if (!fragment2.codec) {
|
|
965
|
-
throw new Error(
|
|
966
|
-
`Fragment "${fragment2.name}" is missing codec. Lazy fragments must be resolved before encode.`
|
|
967
|
-
);
|
|
968
|
-
}
|
|
969
|
-
const messageData = {
|
|
970
|
-
id: fragment2.id ?? crypto.randomUUID(),
|
|
971
|
-
chatId: this.#chatId,
|
|
972
|
-
parentId,
|
|
973
|
-
name: fragment2.name,
|
|
974
|
-
type: fragment2.type,
|
|
975
|
-
data: fragment2.codec.encode(),
|
|
976
|
-
createdAt: now
|
|
977
|
-
};
|
|
978
|
-
await this.#store.addMessage(messageData);
|
|
979
|
-
parentId = messageData.id;
|
|
980
|
-
}
|
|
981
|
-
await this.#store.updateBranchHead(this.#branch.id, parentId);
|
|
982
|
-
this.#branch.headMessageId = parentId;
|
|
983
|
-
this.#pendingMessages = [];
|
|
984
|
-
}
|
|
985
|
-
/**
|
|
986
|
-
* Resolve a lazy fragment by finding the appropriate ID.
|
|
987
|
-
*/
|
|
988
|
-
async #resolveLazyFragment(fragment2) {
|
|
989
|
-
const lazy = fragment2[LAZY_ID];
|
|
990
|
-
if (lazy.type === "last-assistant") {
|
|
991
|
-
const lastId = await this.#getLastAssistantId();
|
|
992
|
-
return assistantText(lazy.content, { id: lastId ?? crypto.randomUUID() });
|
|
993
|
-
}
|
|
994
|
-
throw new Error(`Unknown lazy fragment type: ${lazy.type}`);
|
|
995
|
-
}
|
|
996
|
-
/**
|
|
997
|
-
* Find the most recent assistant message ID (pending or persisted).
|
|
998
|
-
*/
|
|
999
|
-
async #getLastAssistantId() {
|
|
1000
|
-
for (let i = this.#pendingMessages.length - 1; i >= 0; i--) {
|
|
1001
|
-
const msg = this.#pendingMessages[i];
|
|
1002
|
-
if (msg.name === "assistant" && !isLazyFragment(msg)) {
|
|
1003
|
-
return msg.id;
|
|
1004
|
-
}
|
|
1005
|
-
}
|
|
1006
|
-
if (this.#branch?.headMessageId) {
|
|
1007
|
-
const chain = await this.#store.getMessageChain(
|
|
1008
|
-
this.#branch.headMessageId
|
|
1009
|
-
);
|
|
1010
|
-
for (let i = chain.length - 1; i >= 0; i--) {
|
|
1011
|
-
if (chain[i].name === "assistant") {
|
|
1012
|
-
return chain[i].id;
|
|
1013
|
-
}
|
|
1014
|
-
}
|
|
1015
|
-
}
|
|
1016
|
-
return void 0;
|
|
1017
|
-
}
|
|
1018
|
-
/**
|
|
1019
|
-
* Estimate token count and cost for the full context.
|
|
1020
|
-
*
|
|
1021
|
-
* Includes:
|
|
1022
|
-
* - System prompt fragments (role, hints, etc.)
|
|
1023
|
-
* - Persisted chat messages (from store)
|
|
1024
|
-
* - Pending messages (not yet saved)
|
|
1025
|
-
*
|
|
1026
|
-
* @param modelId - Model ID (e.g., "openai:gpt-4o", "anthropic:claude-3-5-sonnet")
|
|
1027
|
-
* @param options - Optional settings
|
|
1028
|
-
* @returns Estimate result with token counts, costs, and per-fragment breakdown
|
|
1029
|
-
*/
|
|
1030
|
-
async estimate(modelId, options = {}) {
|
|
1031
|
-
await this.#ensureInitialized();
|
|
1032
|
-
const renderer = options.renderer ?? new XmlRenderer();
|
|
1033
|
-
const registry = getModelsRegistry();
|
|
1034
|
-
await registry.load();
|
|
1035
|
-
const model = registry.get(modelId);
|
|
1036
|
-
if (!model) {
|
|
1037
|
-
throw new Error(
|
|
1038
|
-
`Model "${modelId}" not found. Call load() first or check model ID.`
|
|
1039
|
-
);
|
|
1040
|
-
}
|
|
1041
|
-
const tokenizer = registry.getTokenizer(modelId);
|
|
1042
|
-
const fragmentEstimates = [];
|
|
1043
|
-
for (const fragment2 of this.#fragments) {
|
|
1044
|
-
const rendered = renderer.render([fragment2]);
|
|
1045
|
-
const tokens = tokenizer.count(rendered);
|
|
1046
|
-
const cost = tokens / 1e6 * model.cost.input;
|
|
1047
|
-
fragmentEstimates.push({
|
|
1048
|
-
id: fragment2.id,
|
|
1049
|
-
name: fragment2.name,
|
|
1050
|
-
tokens,
|
|
1051
|
-
cost
|
|
1052
|
-
});
|
|
1053
|
-
}
|
|
1054
|
-
if (this.#branch?.headMessageId) {
|
|
1055
|
-
const chain = await this.#store.getMessageChain(
|
|
1056
|
-
this.#branch.headMessageId
|
|
1057
|
-
);
|
|
1058
|
-
for (const msg of chain) {
|
|
1059
|
-
const content = String(msg.data);
|
|
1060
|
-
const tokens = tokenizer.count(content);
|
|
1061
|
-
const cost = tokens / 1e6 * model.cost.input;
|
|
1062
|
-
fragmentEstimates.push({
|
|
1063
|
-
name: msg.name,
|
|
1064
|
-
id: msg.id,
|
|
1065
|
-
tokens,
|
|
1066
|
-
cost
|
|
1067
|
-
});
|
|
1068
|
-
}
|
|
1069
|
-
}
|
|
1070
|
-
for (const fragment2 of this.#pendingMessages) {
|
|
1071
|
-
const content = String(fragment2.data);
|
|
1072
|
-
const tokens = tokenizer.count(content);
|
|
1073
|
-
const cost = tokens / 1e6 * model.cost.input;
|
|
1074
|
-
fragmentEstimates.push({
|
|
1075
|
-
name: fragment2.name,
|
|
1076
|
-
id: fragment2.id,
|
|
1077
|
-
tokens,
|
|
1078
|
-
cost
|
|
1079
|
-
});
|
|
1080
|
-
}
|
|
1081
|
-
const totalTokens = fragmentEstimates.reduce((sum, f) => sum + f.tokens, 0);
|
|
1082
|
-
const totalCost = fragmentEstimates.reduce((sum, f) => sum + f.cost, 0);
|
|
1083
|
-
return {
|
|
1084
|
-
model: model.id,
|
|
1085
|
-
provider: model.provider,
|
|
1086
|
-
tokens: totalTokens,
|
|
1087
|
-
cost: totalCost,
|
|
1088
|
-
limits: {
|
|
1089
|
-
context: model.limit.context,
|
|
1090
|
-
output: model.limit.output,
|
|
1091
|
-
exceedsContext: totalTokens > model.limit.context
|
|
1092
|
-
},
|
|
1093
|
-
fragments: fragmentEstimates
|
|
1094
|
-
};
|
|
1095
|
-
}
|
|
1096
|
-
/**
|
|
1097
|
-
* Rewind to a specific message by ID.
|
|
1098
|
-
*
|
|
1099
|
-
* Creates a new branch from that message, preserving the original branch.
|
|
1100
|
-
* The new branch becomes active.
|
|
1101
|
-
*
|
|
1102
|
-
* @param messageId - The message ID to rewind to
|
|
1103
|
-
* @returns The new branch info
|
|
1104
|
-
*
|
|
1105
|
-
* @example
|
|
1106
|
-
* ```ts
|
|
1107
|
-
* context.set(user('What is 2 + 2?', { id: 'q1' }));
|
|
1108
|
-
* context.set(assistant('The answer is 5.', { id: 'wrong' })); // Oops!
|
|
1109
|
-
* await context.save();
|
|
1110
|
-
*
|
|
1111
|
-
* // Rewind to the question, creates new branch
|
|
1112
|
-
* const newBranch = await context.rewind('q1');
|
|
1113
|
-
*
|
|
1114
|
-
* // Now add correct answer on new branch
|
|
1115
|
-
* context.set(assistant('The answer is 4.'));
|
|
1116
|
-
* await context.save();
|
|
1117
|
-
* ```
|
|
1118
|
-
*/
|
|
1119
|
-
async rewind(messageId) {
|
|
1120
|
-
await this.#ensureInitialized();
|
|
1121
|
-
const message2 = await this.#store.getMessage(messageId);
|
|
1122
|
-
if (!message2) {
|
|
1123
|
-
throw new Error(`Message "${messageId}" not found`);
|
|
1124
|
-
}
|
|
1125
|
-
if (message2.chatId !== this.#chatId) {
|
|
1126
|
-
throw new Error(`Message "${messageId}" belongs to a different chat`);
|
|
1127
|
-
}
|
|
1128
|
-
return this.#createBranchFrom(messageId, true);
|
|
1129
|
-
}
|
|
1130
|
-
/**
|
|
1131
|
-
* Create a checkpoint at the current position.
|
|
1132
|
-
*
|
|
1133
|
-
* A checkpoint is a named pointer to the current branch head.
|
|
1134
|
-
* Use restore() to return to this point later.
|
|
1135
|
-
*
|
|
1136
|
-
* @param name - Name for the checkpoint
|
|
1137
|
-
* @returns The checkpoint info
|
|
1138
|
-
*
|
|
1139
|
-
* @example
|
|
1140
|
-
* ```ts
|
|
1141
|
-
* context.set(user('I want to learn a new skill.'));
|
|
1142
|
-
* context.set(assistant('Would you like coding or cooking?'));
|
|
1143
|
-
* await context.save();
|
|
1144
|
-
*
|
|
1145
|
-
* // Save checkpoint before user's choice
|
|
1146
|
-
* const cp = await context.checkpoint('before-choice');
|
|
1147
|
-
* ```
|
|
1148
|
-
*/
|
|
1149
|
-
async checkpoint(name) {
|
|
1150
|
-
await this.#ensureInitialized();
|
|
1151
|
-
if (!this.#branch?.headMessageId) {
|
|
1152
|
-
throw new Error("Cannot create checkpoint: no messages in conversation");
|
|
1153
|
-
}
|
|
1154
|
-
const checkpoint = {
|
|
1155
|
-
id: crypto.randomUUID(),
|
|
1156
|
-
chatId: this.#chatId,
|
|
1157
|
-
name,
|
|
1158
|
-
messageId: this.#branch.headMessageId,
|
|
1159
|
-
createdAt: Date.now()
|
|
1160
|
-
};
|
|
1161
|
-
await this.#store.createCheckpoint(checkpoint);
|
|
1162
|
-
return {
|
|
1163
|
-
id: checkpoint.id,
|
|
1164
|
-
name: checkpoint.name,
|
|
1165
|
-
messageId: checkpoint.messageId,
|
|
1166
|
-
createdAt: checkpoint.createdAt
|
|
1167
|
-
};
|
|
1168
|
-
}
|
|
1169
|
-
/**
|
|
1170
|
-
* Restore to a checkpoint by creating a new branch from that point.
|
|
1171
|
-
*
|
|
1172
|
-
* @param name - Name of the checkpoint to restore
|
|
1173
|
-
* @returns The new branch info
|
|
1174
|
-
*
|
|
1175
|
-
* @example
|
|
1176
|
-
* ```ts
|
|
1177
|
-
* // User chose cooking, but wants to try coding path
|
|
1178
|
-
* await context.restore('before-choice');
|
|
1179
|
-
*
|
|
1180
|
-
* context.set(user('I want to learn coding.'));
|
|
1181
|
-
* context.set(assistant('Python is a great starting language!'));
|
|
1182
|
-
* await context.save();
|
|
1183
|
-
* ```
|
|
1184
|
-
*/
|
|
1185
|
-
async restore(name) {
|
|
1186
|
-
await this.#ensureInitialized();
|
|
1187
|
-
const checkpoint = await this.#store.getCheckpoint(this.#chatId, name);
|
|
1188
|
-
if (!checkpoint) {
|
|
1189
|
-
throw new Error(
|
|
1190
|
-
`Checkpoint "${name}" not found in chat "${this.#chatId}"`
|
|
1191
|
-
);
|
|
1192
|
-
}
|
|
1193
|
-
return this.rewind(checkpoint.messageId);
|
|
1194
|
-
}
|
|
1195
|
-
/**
|
|
1196
|
-
* Switch to a different branch by name.
|
|
1197
|
-
*
|
|
1198
|
-
* @param name - Branch name to switch to
|
|
1199
|
-
*
|
|
1200
|
-
* @example
|
|
1201
|
-
* ```ts
|
|
1202
|
-
* // List branches (via store)
|
|
1203
|
-
* const branches = await store.listBranches(context.chatId);
|
|
1204
|
-
* console.log(branches); // [{name: 'main', ...}, {name: 'main-v2', ...}]
|
|
1205
|
-
*
|
|
1206
|
-
* // Switch to original branch
|
|
1207
|
-
* await context.switchBranch('main');
|
|
1208
|
-
* ```
|
|
1209
|
-
*/
|
|
1210
|
-
async switchBranch(name) {
|
|
1211
|
-
await this.#ensureInitialized();
|
|
1212
|
-
const branch = await this.#store.getBranch(this.#chatId, name);
|
|
1213
|
-
if (!branch) {
|
|
1214
|
-
throw new Error(`Branch "${name}" not found in chat "${this.#chatId}"`);
|
|
1215
|
-
}
|
|
1216
|
-
await this.#store.setActiveBranch(this.#chatId, branch.id);
|
|
1217
|
-
this.#branch = { ...branch, isActive: true };
|
|
1218
|
-
this.#branchName = name;
|
|
1219
|
-
this.#pendingMessages = [];
|
|
1220
|
-
}
|
|
1221
|
-
/**
|
|
1222
|
-
* Create a parallel branch from the current position ("by the way").
|
|
1223
|
-
*
|
|
1224
|
-
* Use this when you want to fork the conversation without leaving
|
|
1225
|
-
* the current branch. Common use case: user wants to ask another
|
|
1226
|
-
* question while waiting for the model to respond.
|
|
1227
|
-
*
|
|
1228
|
-
* Unlike rewind(), this method:
|
|
1229
|
-
* - Uses the current HEAD (no messageId needed)
|
|
1230
|
-
* - Does NOT switch to the new branch
|
|
1231
|
-
* - Keeps pending messages intact
|
|
1232
|
-
*
|
|
1233
|
-
* @returns The new branch info (does not switch to it)
|
|
1234
|
-
* @throws Error if no messages exist in the conversation
|
|
1235
|
-
*
|
|
1236
|
-
* @example
|
|
1237
|
-
* ```ts
|
|
1238
|
-
* // User asked a question, model is generating...
|
|
1239
|
-
* context.set(user('What is the weather?'));
|
|
1240
|
-
* await context.save();
|
|
1241
|
-
*
|
|
1242
|
-
* // User wants to ask something else without waiting
|
|
1243
|
-
* const newBranch = await context.btw();
|
|
1244
|
-
* // newBranch = { name: 'main-v2', ... }
|
|
1245
|
-
*
|
|
1246
|
-
* // Later, switch to the new branch and add the question
|
|
1247
|
-
* await context.switchBranch(newBranch.name);
|
|
1248
|
-
* context.set(user('Also, what time is it?'));
|
|
1249
|
-
* await context.save();
|
|
1250
|
-
* ```
|
|
1251
|
-
*/
|
|
1252
|
-
async btw() {
|
|
1253
|
-
await this.#ensureInitialized();
|
|
1254
|
-
if (!this.#branch?.headMessageId) {
|
|
1255
|
-
throw new Error("Cannot create btw branch: no messages in conversation");
|
|
1256
|
-
}
|
|
1257
|
-
return this.#createBranchFrom(this.#branch.headMessageId, false);
|
|
1258
|
-
}
|
|
1259
|
-
/**
|
|
1260
|
-
* Update metadata for the current chat.
|
|
1261
|
-
*
|
|
1262
|
-
* @param updates - Partial metadata to merge (title, metadata)
|
|
1263
|
-
*
|
|
1264
|
-
* @example
|
|
1265
|
-
* ```ts
|
|
1266
|
-
* await context.updateChat({
|
|
1267
|
-
* title: 'Coding Help Session',
|
|
1268
|
-
* metadata: { tags: ['python', 'debugging'] }
|
|
1269
|
-
* });
|
|
1270
|
-
* ```
|
|
1271
|
-
*/
|
|
1272
|
-
async updateChat(updates) {
|
|
1273
|
-
await this.#ensureInitialized();
|
|
1274
|
-
const storeUpdates = {};
|
|
1275
|
-
if (updates.title !== void 0) {
|
|
1276
|
-
storeUpdates.title = updates.title;
|
|
1277
|
-
}
|
|
1278
|
-
if (updates.metadata !== void 0) {
|
|
1279
|
-
storeUpdates.metadata = {
|
|
1280
|
-
...this.#chatData?.metadata,
|
|
1281
|
-
...updates.metadata
|
|
1282
|
-
};
|
|
1283
|
-
}
|
|
1284
|
-
this.#chatData = await this.#store.updateChat(this.#chatId, storeUpdates);
|
|
1285
|
-
}
|
|
1286
|
-
/**
|
|
1287
|
-
* Track token usage for the current chat.
|
|
1288
|
-
* Accumulates usage metrics in chat.metadata.usage.
|
|
1289
|
-
*
|
|
1290
|
-
* @param usage - Token usage from AI SDK (LanguageModelUsage)
|
|
1291
|
-
*
|
|
1292
|
-
* @example
|
|
1293
|
-
* ```ts
|
|
1294
|
-
* // In onFinish callback
|
|
1295
|
-
* const usage = await result.totalUsage;
|
|
1296
|
-
* await context.trackUsage(usage);
|
|
1297
|
-
* ```
|
|
1298
|
-
*/
|
|
1299
|
-
async trackUsage(usage) {
|
|
1300
|
-
await this.#ensureInitialized();
|
|
1301
|
-
const freshChatData = await this.#store.getChat(this.#chatId);
|
|
1302
|
-
const currentUsage = freshChatData?.metadata?.usage ?? {};
|
|
1303
|
-
const updatedUsage = mergeWith(
|
|
1304
|
-
{},
|
|
1305
|
-
currentUsage,
|
|
1306
|
-
usage,
|
|
1307
|
-
(a, b) => typeof a === "number" || typeof b === "number" ? (a ?? 0) + (b ?? 0) : void 0
|
|
1308
|
-
);
|
|
1309
|
-
this.#chatData = await this.#store.updateChat(this.#chatId, {
|
|
1310
|
-
metadata: {
|
|
1311
|
-
...freshChatData?.metadata,
|
|
1312
|
-
usage: updatedUsage
|
|
1313
|
-
}
|
|
1314
|
-
});
|
|
1315
|
-
}
|
|
1316
|
-
/**
|
|
1317
|
-
* Consolidate context fragments (no-op for now).
|
|
1318
|
-
*
|
|
1319
|
-
* This is a placeholder for future functionality that merges context fragments
|
|
1320
|
-
* using specific rules. Currently, it does nothing.
|
|
1321
|
-
*
|
|
1322
|
-
* @experimental
|
|
1323
|
-
*/
|
|
1324
|
-
consolidate() {
|
|
1325
|
-
return void 0;
|
|
1326
|
-
}
|
|
1327
|
-
/**
|
|
1328
|
-
* Extract skill mounts from available_skills fragments.
|
|
1329
|
-
* Returns unified mount array where entries with `name` are individual skills.
|
|
1330
|
-
*
|
|
1331
|
-
* @example
|
|
1332
|
-
* ```ts
|
|
1333
|
-
* const context = new ContextEngine({ store, chatId, userId })
|
|
1334
|
-
* .set(skills({ paths: [{ host: './skills', sandbox: '/skills' }] }));
|
|
1335
|
-
*
|
|
1336
|
-
* const { mounts } = context.getSkillMounts();
|
|
1337
|
-
* // mounts: [{ name: 'bi-dashboards', host: './skills/bi-dashboards/SKILL.md', sandbox: '/skills/bi-dashboards/SKILL.md' }]
|
|
1338
|
-
*
|
|
1339
|
-
* // Extract skills only (entries with name)
|
|
1340
|
-
* const skills = mounts.filter(m => m.name);
|
|
1341
|
-
* ```
|
|
1342
|
-
*/
|
|
1343
|
-
getSkillMounts() {
|
|
1344
|
-
for (const fragment2 of this.#fragments) {
|
|
1345
|
-
if (fragment2.name === "available_skills" && fragment2.metadata?.mounts) {
|
|
1346
|
-
return { mounts: fragment2.metadata.mounts };
|
|
1347
|
-
}
|
|
1348
|
-
}
|
|
1349
|
-
return { mounts: [] };
|
|
1350
|
-
}
|
|
1351
|
-
/**
|
|
1352
|
-
* Inspect the full context state for debugging.
|
|
1353
|
-
* Returns a JSON-serializable object with context information.
|
|
1354
|
-
*
|
|
1355
|
-
* @param options - Inspection options (modelId and renderer required)
|
|
1356
|
-
* @returns Complete inspection data including estimates, rendered output, fragments, and graph
|
|
1357
|
-
*
|
|
1358
|
-
* @example
|
|
1359
|
-
* ```ts
|
|
1360
|
-
* const inspection = await context.inspect({
|
|
1361
|
-
* modelId: 'openai:gpt-4o',
|
|
1362
|
-
* renderer: new XmlRenderer(),
|
|
1363
|
-
* });
|
|
1364
|
-
* console.log(JSON.stringify(inspection, null, 2));
|
|
1365
|
-
*
|
|
1366
|
-
* // Or write to file for analysis
|
|
1367
|
-
* await fs.writeFile('context-debug.json', JSON.stringify(inspection, null, 2));
|
|
1368
|
-
* ```
|
|
1369
|
-
*/
|
|
1370
|
-
async inspect(options) {
|
|
1371
|
-
await this.#ensureInitialized();
|
|
1372
|
-
const { renderer } = options;
|
|
1373
|
-
const estimateResult = await this.estimate(options.modelId, { renderer });
|
|
1374
|
-
const rendered = renderer.render(this.#fragments);
|
|
1375
|
-
const persistedMessages = [];
|
|
1376
|
-
if (this.#branch?.headMessageId) {
|
|
1377
|
-
const chain = await this.#store.getMessageChain(
|
|
1378
|
-
this.#branch.headMessageId
|
|
1379
|
-
);
|
|
1380
|
-
persistedMessages.push(...chain);
|
|
1381
|
-
}
|
|
1382
|
-
const graph = await this.#store.getGraph(this.#chatId);
|
|
1383
|
-
return {
|
|
1384
|
-
estimate: estimateResult,
|
|
1385
|
-
rendered,
|
|
1386
|
-
fragments: {
|
|
1387
|
-
context: [...this.#fragments],
|
|
1388
|
-
pending: [...this.#pendingMessages],
|
|
1389
|
-
persisted: persistedMessages
|
|
1390
|
-
},
|
|
1391
|
-
graph,
|
|
1392
|
-
meta: {
|
|
1393
|
-
chatId: this.#chatId,
|
|
1394
|
-
branch: this.#branchName,
|
|
1395
|
-
timestamp: Date.now()
|
|
1396
|
-
}
|
|
1397
|
-
};
|
|
1398
|
-
}
|
|
1399
|
-
};
|
|
1400
|
-
function term(name, definition) {
|
|
1401
|
-
return {
|
|
1402
|
-
name: "term",
|
|
1403
|
-
data: { name, definition }
|
|
1404
|
-
};
|
|
1405
|
-
}
|
|
1406
|
-
function hint(text) {
|
|
1407
|
-
return {
|
|
1408
|
-
name: "hint",
|
|
1409
|
-
data: text
|
|
1410
|
-
};
|
|
1411
|
-
}
|
|
1412
|
-
function guardrail(input) {
|
|
1413
|
-
return {
|
|
1414
|
-
name: "guardrail",
|
|
1415
|
-
data: {
|
|
1416
|
-
rule: input.rule,
|
|
1417
|
-
...input.reason && { reason: input.reason },
|
|
1418
|
-
...input.action && { action: input.action }
|
|
1419
|
-
}
|
|
1420
|
-
};
|
|
1421
|
-
}
|
|
1422
|
-
function explain(input) {
|
|
1423
|
-
return {
|
|
1424
|
-
name: "explain",
|
|
1425
|
-
data: {
|
|
1426
|
-
concept: input.concept,
|
|
1427
|
-
explanation: input.explanation,
|
|
1428
|
-
...input.therefore && { therefore: input.therefore }
|
|
1429
|
-
}
|
|
1430
|
-
};
|
|
1431
|
-
}
|
|
1432
|
-
function example(input) {
|
|
1433
|
-
return {
|
|
1434
|
-
name: "example",
|
|
1435
|
-
data: {
|
|
1436
|
-
question: input.question,
|
|
1437
|
-
answer: input.answer,
|
|
1438
|
-
...input.note && { note: input.note }
|
|
1439
|
-
}
|
|
1440
|
-
};
|
|
1441
|
-
}
|
|
1442
|
-
function clarification(input) {
|
|
1443
|
-
return {
|
|
1444
|
-
name: "clarification",
|
|
1445
|
-
data: {
|
|
1446
|
-
when: input.when,
|
|
1447
|
-
ask: input.ask,
|
|
1448
|
-
reason: input.reason
|
|
1449
|
-
}
|
|
1450
|
-
};
|
|
1451
|
-
}
|
|
1452
|
-
function workflow(input) {
|
|
1453
|
-
return {
|
|
1454
|
-
name: "workflow",
|
|
1455
|
-
data: {
|
|
1456
|
-
task: input.task,
|
|
1457
|
-
steps: input.steps,
|
|
1458
|
-
...input.triggers?.length && { triggers: input.triggers },
|
|
1459
|
-
...input.notes && { notes: input.notes }
|
|
1460
|
-
}
|
|
1461
|
-
};
|
|
1462
|
-
}
|
|
1463
|
-
function quirk(input) {
|
|
1464
|
-
return {
|
|
1465
|
-
name: "quirk",
|
|
1466
|
-
data: {
|
|
1467
|
-
issue: input.issue,
|
|
1468
|
-
workaround: input.workaround
|
|
1469
|
-
}
|
|
1470
|
-
};
|
|
1471
|
-
}
|
|
1472
|
-
function styleGuide(input) {
|
|
1473
|
-
return {
|
|
1474
|
-
name: "styleGuide",
|
|
1475
|
-
data: {
|
|
1476
|
-
prefer: input.prefer,
|
|
1477
|
-
...input.never && { never: input.never },
|
|
1478
|
-
...input.always && { always: input.always }
|
|
1479
|
-
}
|
|
1480
|
-
};
|
|
1481
|
-
}
|
|
1482
|
-
function analogy(input) {
|
|
1483
|
-
return {
|
|
1484
|
-
name: "analogy",
|
|
1485
|
-
data: {
|
|
1486
|
-
concepts: input.concepts,
|
|
1487
|
-
relationship: input.relationship,
|
|
1488
|
-
...input.insight && { insight: input.insight },
|
|
1489
|
-
...input.therefore && { therefore: input.therefore },
|
|
1490
|
-
...input.pitfall && { pitfall: input.pitfall }
|
|
1491
|
-
}
|
|
1492
|
-
};
|
|
1493
|
-
}
|
|
1494
|
-
function persona(input) {
|
|
1495
|
-
return {
|
|
1496
|
-
name: "persona",
|
|
1497
|
-
data: {
|
|
1498
|
-
name: input.name,
|
|
1499
|
-
...input.role && { role: input.role },
|
|
1500
|
-
...input.objective && { objective: input.objective },
|
|
1501
|
-
...input.tone && { tone: input.tone }
|
|
1502
|
-
}
|
|
1503
|
-
};
|
|
1504
|
-
}
|
|
1505
|
-
var SKILLS_INSTRUCTIONS = dedent`A skill is a set of local instructions to follow that is stored in a \`SKILL.md\` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
|
|
1506
|
-
|
|
1507
|
-
### How to use skills
|
|
1508
|
-
- Discovery: The list below shows the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
|
|
1509
|
-
- Trigger rules: If the user names a skill (with \`$SkillName\` or plain text) OR the task clearly matches a skill's description shown below, you must use that skill for that turn before doing anything else. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
|
|
1510
|
-
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
|
|
1511
|
-
- How to use a skill (progressive disclosure):
|
|
1512
|
-
1) After deciding to use a skill, open its \`SKILL.md\`. Read only enough to follow the workflow.
|
|
1513
|
-
2) If \`SKILL.md\` points to extra folders such as \`references/\`, load only the specific files needed for the request; don't bulk-load everything.
|
|
1514
|
-
3) If \`scripts/\` exist, prefer running or patching them instead of retyping large code blocks.
|
|
1515
|
-
4) If \`assets/\` or templates exist, reuse them instead of recreating from scratch.
|
|
1516
|
-
- Coordination and sequencing:
|
|
1517
|
-
- If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
|
|
1518
|
-
- Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
|
|
1519
|
-
- Context hygiene:
|
|
1520
|
-
- Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
|
|
1521
|
-
- Avoid deep reference-chasing: prefer opening only files directly linked from \`SKILL.md\` unless you're blocked.
|
|
1522
|
-
- When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
|
|
1523
|
-
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.`;
|
|
1524
|
-
var ddl_sqlite_default = "-- Context Store DDL for SQLite\n-- This schema implements a DAG-based message history with branching and checkpoints.\n\n-- Performance PRAGMAs (session-level, run on each connection)\nPRAGMA journal_mode = WAL;\nPRAGMA synchronous = NORMAL;\nPRAGMA cache_size = -64000;\nPRAGMA temp_store = MEMORY;\nPRAGMA mmap_size = 268435456;\n\n-- Integrity\nPRAGMA foreign_keys = ON;\n\n-- Chats table\n-- createdAt/updatedAt: DEFAULT for insert, inline SET for updates\nCREATE TABLE IF NOT EXISTS chats (\n id TEXT PRIMARY KEY,\n userId TEXT NOT NULL,\n title TEXT,\n metadata TEXT,\n createdAt INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),\n updatedAt INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000)\n);\n\nCREATE INDEX IF NOT EXISTS idx_chats_updatedAt ON chats(updatedAt);\nCREATE INDEX IF NOT EXISTS idx_chats_userId ON chats(userId);\n-- Composite index for listChats(): WHERE userId = ? ORDER BY updatedAt DESC\nCREATE INDEX IF NOT EXISTS idx_chats_userId_updatedAt ON chats(userId, updatedAt DESC);\n\n-- Messages table (nodes in the DAG)\nCREATE TABLE IF NOT EXISTS messages (\n id TEXT PRIMARY KEY,\n chatId TEXT NOT NULL,\n parentId TEXT,\n name TEXT NOT NULL,\n type TEXT,\n data TEXT NOT NULL,\n createdAt INTEGER NOT NULL,\n FOREIGN KEY (chatId) REFERENCES chats(id) ON DELETE CASCADE,\n FOREIGN KEY (parentId) REFERENCES messages(id)\n);\n\nCREATE INDEX IF NOT EXISTS idx_messages_chatId ON messages(chatId);\nCREATE INDEX IF NOT EXISTS idx_messages_parentId ON messages(parentId);\n-- Composite index for recursive CTE parent traversal in getMessageChain()\nCREATE INDEX IF NOT EXISTS idx_messages_chatId_parentId ON messages(chatId, parentId);\n\n-- Branches table (pointers to head messages)\nCREATE TABLE IF NOT EXISTS branches (\n id TEXT PRIMARY KEY,\n chatId TEXT NOT NULL,\n name TEXT NOT NULL,\n headMessageId TEXT,\n isActive INTEGER NOT NULL DEFAULT 0,\n createdAt INTEGER NOT NULL,\n FOREIGN KEY (chatId) REFERENCES chats(id) ON DELETE CASCADE,\n FOREIGN KEY (headMessageId) REFERENCES messages(id),\n UNIQUE(chatId, name)\n);\n\nCREATE INDEX IF NOT EXISTS idx_branches_chatId ON branches(chatId);\n-- Composite index for getActiveBranch(): WHERE chatId = ? AND isActive = 1\nCREATE INDEX IF NOT EXISTS idx_branches_chatId_isActive ON branches(chatId, isActive);\n\n-- Checkpoints table (pointers to message nodes)\nCREATE TABLE IF NOT EXISTS checkpoints (\n id TEXT PRIMARY KEY,\n chatId TEXT NOT NULL,\n name TEXT NOT NULL,\n messageId TEXT NOT NULL,\n createdAt INTEGER NOT NULL,\n FOREIGN KEY (chatId) REFERENCES chats(id) ON DELETE CASCADE,\n FOREIGN KEY (messageId) REFERENCES messages(id),\n UNIQUE(chatId, name)\n);\n\nCREATE INDEX IF NOT EXISTS idx_checkpoints_chatId ON checkpoints(chatId);\n\n-- FTS5 virtual table for full-text search\n-- messageId/chatId/name are UNINDEXED (stored but not searchable, used for filtering/joining)\n-- Only 'content' is indexed for full-text search\nCREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(\n messageId UNINDEXED,\n chatId UNINDEXED,\n name UNINDEXED,\n content,\n tokenize='porter unicode61'\n);\n";
|
|
1525
|
-
var SqliteContextStore = class extends ContextStore {
|
|
1526
|
-
#db;
|
|
1527
|
-
#statements = /* @__PURE__ */ new Map();
|
|
1528
|
-
/**
|
|
1529
|
-
* Get or create a prepared statement.
|
|
1530
|
-
* Statements are cached for the lifetime of the store to avoid
|
|
1531
|
-
* repeated SQL parsing and compilation overhead.
|
|
1532
|
-
*/
|
|
1533
|
-
#stmt(sql) {
|
|
1534
|
-
let stmt = this.#statements.get(sql);
|
|
1535
|
-
if (!stmt) {
|
|
1536
|
-
stmt = this.#db.prepare(sql);
|
|
1537
|
-
this.#statements.set(sql, stmt);
|
|
1538
|
-
}
|
|
1539
|
-
return stmt;
|
|
1540
|
-
}
|
|
1541
|
-
constructor(path3) {
|
|
1542
|
-
super();
|
|
1543
|
-
this.#db = new DatabaseSync(path3);
|
|
1544
|
-
this.#db.exec(ddl_sqlite_default);
|
|
1545
|
-
}
|
|
1546
|
-
/**
|
|
1547
|
-
* Execute a function within a transaction.
|
|
1548
|
-
* Automatically commits on success or rolls back on error.
|
|
1549
|
-
*/
|
|
1550
|
-
#useTransaction(fn) {
|
|
1551
|
-
this.#db.exec("BEGIN TRANSACTION");
|
|
1552
|
-
try {
|
|
1553
|
-
const result = fn();
|
|
1554
|
-
this.#db.exec("COMMIT");
|
|
1555
|
-
return result;
|
|
1556
|
-
} catch (error) {
|
|
1557
|
-
this.#db.exec("ROLLBACK");
|
|
1558
|
-
throw error;
|
|
1559
|
-
}
|
|
1560
|
-
}
|
|
1561
|
-
// ==========================================================================
|
|
1562
|
-
// Chat Operations
|
|
1563
|
-
// ==========================================================================
|
|
1564
|
-
async createChat(chat) {
|
|
1565
|
-
return this.#useTransaction(() => {
|
|
1566
|
-
const row = this.#db.prepare(
|
|
1567
|
-
`INSERT INTO chats (id, userId, title, metadata)
|
|
1568
|
-
VALUES (?, ?, ?, ?)
|
|
1569
|
-
RETURNING *`
|
|
1570
|
-
).get(
|
|
1571
|
-
chat.id,
|
|
1572
|
-
chat.userId,
|
|
1573
|
-
chat.title ?? null,
|
|
1574
|
-
chat.metadata ? JSON.stringify(chat.metadata) : null
|
|
1575
|
-
);
|
|
1576
|
-
this.#db.prepare(
|
|
1577
|
-
`INSERT INTO branches (id, chatId, name, headMessageId, isActive, createdAt)
|
|
1578
|
-
VALUES (?, ?, 'main', NULL, 1, ?)`
|
|
1579
|
-
).run(crypto.randomUUID(), chat.id, Date.now());
|
|
1580
|
-
return {
|
|
1581
|
-
id: row.id,
|
|
1582
|
-
userId: row.userId,
|
|
1583
|
-
title: row.title ?? void 0,
|
|
1584
|
-
metadata: row.metadata ? JSON.parse(row.metadata) : void 0,
|
|
1585
|
-
createdAt: row.createdAt,
|
|
1586
|
-
updatedAt: row.updatedAt
|
|
1587
|
-
};
|
|
1588
|
-
});
|
|
1589
|
-
}
|
|
1590
|
-
async upsertChat(chat) {
|
|
1591
|
-
return this.#useTransaction(() => {
|
|
1592
|
-
const row = this.#db.prepare(
|
|
1593
|
-
`INSERT INTO chats (id, userId, title, metadata)
|
|
1594
|
-
VALUES (?, ?, ?, ?)
|
|
1595
|
-
ON CONFLICT(id) DO UPDATE SET id = excluded.id
|
|
1596
|
-
RETURNING *`
|
|
1597
|
-
).get(
|
|
1598
|
-
chat.id,
|
|
1599
|
-
chat.userId,
|
|
1600
|
-
chat.title ?? null,
|
|
1601
|
-
chat.metadata ? JSON.stringify(chat.metadata) : null
|
|
1602
|
-
);
|
|
1603
|
-
this.#db.prepare(
|
|
1604
|
-
`INSERT OR IGNORE INTO branches (id, chatId, name, headMessageId, isActive, createdAt)
|
|
1605
|
-
VALUES (?, ?, 'main', NULL, 1, ?)`
|
|
1606
|
-
).run(crypto.randomUUID(), chat.id, Date.now());
|
|
1607
|
-
return {
|
|
1608
|
-
id: row.id,
|
|
1609
|
-
userId: row.userId,
|
|
1610
|
-
title: row.title ?? void 0,
|
|
1611
|
-
metadata: row.metadata ? JSON.parse(row.metadata) : void 0,
|
|
1612
|
-
createdAt: row.createdAt,
|
|
1613
|
-
updatedAt: row.updatedAt
|
|
1614
|
-
};
|
|
1615
|
-
});
|
|
1616
|
-
}
|
|
1617
|
-
async getChat(chatId) {
|
|
1618
|
-
const row = this.#db.prepare("SELECT * FROM chats WHERE id = ?").get(chatId);
|
|
1619
|
-
if (!row) {
|
|
1620
|
-
return void 0;
|
|
1621
|
-
}
|
|
1622
|
-
return {
|
|
1623
|
-
id: row.id,
|
|
1624
|
-
userId: row.userId,
|
|
1625
|
-
title: row.title ?? void 0,
|
|
1626
|
-
metadata: row.metadata ? JSON.parse(row.metadata) : void 0,
|
|
1627
|
-
createdAt: row.createdAt,
|
|
1628
|
-
updatedAt: row.updatedAt
|
|
1629
|
-
};
|
|
1630
|
-
}
|
|
1631
|
-
async updateChat(chatId, updates) {
|
|
1632
|
-
const setClauses = ["updatedAt = strftime('%s', 'now') * 1000"];
|
|
1633
|
-
const params = [];
|
|
1634
|
-
if (updates.title !== void 0) {
|
|
1635
|
-
setClauses.push("title = ?");
|
|
1636
|
-
params.push(updates.title ?? null);
|
|
1637
|
-
}
|
|
1638
|
-
if (updates.metadata !== void 0) {
|
|
1639
|
-
setClauses.push("metadata = ?");
|
|
1640
|
-
params.push(JSON.stringify(updates.metadata));
|
|
1641
|
-
}
|
|
1642
|
-
params.push(chatId);
|
|
1643
|
-
const row = this.#db.prepare(
|
|
1644
|
-
`UPDATE chats SET ${setClauses.join(", ")} WHERE id = ? RETURNING *`
|
|
1645
|
-
).get(...params);
|
|
1646
|
-
return {
|
|
1647
|
-
id: row.id,
|
|
1648
|
-
userId: row.userId,
|
|
1649
|
-
title: row.title ?? void 0,
|
|
1650
|
-
metadata: row.metadata ? JSON.parse(row.metadata) : void 0,
|
|
1651
|
-
createdAt: row.createdAt,
|
|
1652
|
-
updatedAt: row.updatedAt
|
|
1653
|
-
};
|
|
1654
|
-
}
|
|
1655
|
-
async listChats(options) {
|
|
1656
|
-
const params = [];
|
|
1657
|
-
const whereClauses = [];
|
|
1658
|
-
let limitClause = "";
|
|
1659
|
-
if (options?.userId) {
|
|
1660
|
-
whereClauses.push("c.userId = ?");
|
|
1661
|
-
params.push(options.userId);
|
|
1662
|
-
}
|
|
1663
|
-
if (options?.metadata) {
|
|
1664
|
-
whereClauses.push(`json_extract(c.metadata, '$.' || ?) = ?`);
|
|
1665
|
-
params.push(options.metadata.key);
|
|
1666
|
-
params.push(
|
|
1667
|
-
typeof options.metadata.value === "boolean" ? options.metadata.value ? 1 : 0 : options.metadata.value
|
|
1668
|
-
);
|
|
1669
|
-
}
|
|
1670
|
-
const whereClause = whereClauses.length > 0 ? `WHERE ${whereClauses.join(" AND ")}` : "";
|
|
1671
|
-
if (options?.limit !== void 0) {
|
|
1672
|
-
limitClause = " LIMIT ?";
|
|
1673
|
-
params.push(options.limit);
|
|
1674
|
-
if (options.offset !== void 0) {
|
|
1675
|
-
limitClause += " OFFSET ?";
|
|
1676
|
-
params.push(options.offset);
|
|
1677
|
-
}
|
|
1678
|
-
}
|
|
1679
|
-
const rows = this.#db.prepare(
|
|
1680
|
-
`SELECT
|
|
1681
|
-
c.id,
|
|
1682
|
-
c.userId,
|
|
1683
|
-
c.title,
|
|
1684
|
-
c.metadata,
|
|
1685
|
-
c.createdAt,
|
|
1686
|
-
c.updatedAt,
|
|
1687
|
-
COUNT(DISTINCT m.id) as messageCount,
|
|
1688
|
-
COUNT(DISTINCT b.id) as branchCount
|
|
1689
|
-
FROM chats c
|
|
1690
|
-
LEFT JOIN messages m ON m.chatId = c.id
|
|
1691
|
-
LEFT JOIN branches b ON b.chatId = c.id
|
|
1692
|
-
${whereClause}
|
|
1693
|
-
GROUP BY c.id
|
|
1694
|
-
ORDER BY c.updatedAt DESC${limitClause}`
|
|
1695
|
-
).all(...params);
|
|
1696
|
-
return rows.map((row) => ({
|
|
1697
|
-
id: row.id,
|
|
1698
|
-
userId: row.userId,
|
|
1699
|
-
title: row.title ?? void 0,
|
|
1700
|
-
metadata: row.metadata ? JSON.parse(row.metadata) : void 0,
|
|
1701
|
-
messageCount: row.messageCount,
|
|
1702
|
-
branchCount: row.branchCount,
|
|
1703
|
-
createdAt: row.createdAt,
|
|
1704
|
-
updatedAt: row.updatedAt
|
|
1705
|
-
}));
|
|
1706
|
-
}
|
|
1707
|
-
async deleteChat(chatId, options) {
|
|
1708
|
-
return this.#useTransaction(() => {
|
|
1709
|
-
const messageIds = this.#db.prepare("SELECT id FROM messages WHERE chatId = ?").all(chatId);
|
|
1710
|
-
let sql = "DELETE FROM chats WHERE id = ?";
|
|
1711
|
-
const params = [chatId];
|
|
1712
|
-
if (options?.userId !== void 0) {
|
|
1713
|
-
sql += " AND userId = ?";
|
|
1714
|
-
params.push(options.userId);
|
|
1715
|
-
}
|
|
1716
|
-
const result = this.#db.prepare(sql).run(...params);
|
|
1717
|
-
if (result.changes > 0 && messageIds.length > 0) {
|
|
1718
|
-
const placeholders = messageIds.map(() => "?").join(", ");
|
|
1719
|
-
this.#db.prepare(
|
|
1720
|
-
`DELETE FROM messages_fts WHERE messageId IN (${placeholders})`
|
|
1721
|
-
).run(...messageIds.map((m) => m.id));
|
|
1722
|
-
}
|
|
1723
|
-
return result.changes > 0;
|
|
1724
|
-
});
|
|
1725
|
-
}
|
|
1726
|
-
// ==========================================================================
|
|
1727
|
-
// Message Operations (Graph Nodes)
|
|
1728
|
-
// ==========================================================================
|
|
1729
|
-
async addMessage(message2) {
|
|
1730
|
-
if (message2.parentId === message2.id) {
|
|
1731
|
-
throw new Error(`Message ${message2.id} cannot be its own parent`);
|
|
1732
|
-
}
|
|
1733
|
-
this.#stmt(
|
|
1734
|
-
`INSERT INTO messages (id, chatId, parentId, name, type, data, createdAt)
|
|
1735
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
1736
|
-
ON CONFLICT(id) DO UPDATE SET
|
|
1737
|
-
name = excluded.name,
|
|
1738
|
-
type = excluded.type,
|
|
1739
|
-
data = excluded.data`
|
|
1740
|
-
).run(
|
|
1741
|
-
message2.id,
|
|
1742
|
-
message2.chatId,
|
|
1743
|
-
message2.parentId,
|
|
1744
|
-
message2.name,
|
|
1745
|
-
message2.type ?? null,
|
|
1746
|
-
JSON.stringify(message2.data),
|
|
1747
|
-
message2.createdAt
|
|
1748
|
-
);
|
|
1749
|
-
const content = typeof message2.data === "string" ? message2.data : JSON.stringify(message2.data);
|
|
1750
|
-
this.#stmt(`DELETE FROM messages_fts WHERE messageId = ?`).run(message2.id);
|
|
1751
|
-
this.#stmt(
|
|
1752
|
-
`INSERT INTO messages_fts(messageId, chatId, name, content)
|
|
1753
|
-
VALUES (?, ?, ?, ?)`
|
|
1754
|
-
).run(message2.id, message2.chatId, message2.name, content);
|
|
1755
|
-
}
|
|
1756
|
-
async getMessage(messageId) {
|
|
1757
|
-
const row = this.#stmt("SELECT * FROM messages WHERE id = ?").get(
|
|
1758
|
-
messageId
|
|
1759
|
-
);
|
|
1760
|
-
if (!row) {
|
|
1761
|
-
return void 0;
|
|
1762
|
-
}
|
|
1763
|
-
return {
|
|
1764
|
-
id: row.id,
|
|
1765
|
-
chatId: row.chatId,
|
|
1766
|
-
parentId: row.parentId,
|
|
1767
|
-
name: row.name,
|
|
1768
|
-
type: row.type ?? void 0,
|
|
1769
|
-
data: JSON.parse(row.data),
|
|
1770
|
-
createdAt: row.createdAt
|
|
1771
|
-
};
|
|
1772
|
-
}
|
|
1773
|
-
async getMessageChain(headId) {
|
|
1774
|
-
const rows = this.#stmt(
|
|
1775
|
-
`WITH RECURSIVE chain AS (
|
|
1776
|
-
SELECT *, 0 as depth FROM messages WHERE id = ?
|
|
1777
|
-
UNION ALL
|
|
1778
|
-
SELECT m.*, c.depth + 1 FROM messages m
|
|
1779
|
-
INNER JOIN chain c ON m.id = c.parentId
|
|
1780
|
-
WHERE c.depth < 100000
|
|
1781
|
-
)
|
|
1782
|
-
SELECT * FROM chain
|
|
1783
|
-
ORDER BY depth DESC`
|
|
1784
|
-
).all(headId);
|
|
1785
|
-
return rows.map((row) => ({
|
|
1786
|
-
id: row.id,
|
|
1787
|
-
chatId: row.chatId,
|
|
1788
|
-
parentId: row.parentId,
|
|
1789
|
-
name: row.name,
|
|
1790
|
-
type: row.type ?? void 0,
|
|
1791
|
-
data: JSON.parse(row.data),
|
|
1792
|
-
createdAt: row.createdAt
|
|
1793
|
-
}));
|
|
1794
|
-
}
|
|
1795
|
-
async hasChildren(messageId) {
|
|
1796
|
-
const row = this.#stmt(
|
|
1797
|
-
"SELECT EXISTS(SELECT 1 FROM messages WHERE parentId = ?) as hasChildren"
|
|
1798
|
-
).get(messageId);
|
|
1799
|
-
return row.hasChildren === 1;
|
|
1800
|
-
}
|
|
1801
|
-
async getMessages(chatId) {
|
|
1802
|
-
const chat = await this.getChat(chatId);
|
|
1803
|
-
if (!chat) {
|
|
1804
|
-
throw new Error(`Chat "${chatId}" not found`);
|
|
1805
|
-
}
|
|
1806
|
-
const activeBranch = await this.getActiveBranch(chatId);
|
|
1807
|
-
if (!activeBranch?.headMessageId) {
|
|
1808
|
-
return [];
|
|
1809
|
-
}
|
|
1810
|
-
return this.getMessageChain(activeBranch.headMessageId);
|
|
1811
|
-
}
|
|
1812
|
-
// ==========================================================================
|
|
1813
|
-
// Branch Operations
|
|
1814
|
-
// ==========================================================================
|
|
1815
|
-
async createBranch(branch) {
|
|
1816
|
-
this.#db.prepare(
|
|
1817
|
-
`INSERT INTO branches (id, chatId, name, headMessageId, isActive, createdAt)
|
|
1818
|
-
VALUES (?, ?, ?, ?, ?, ?)`
|
|
1819
|
-
).run(
|
|
1820
|
-
branch.id,
|
|
1821
|
-
branch.chatId,
|
|
1822
|
-
branch.name,
|
|
1823
|
-
branch.headMessageId,
|
|
1824
|
-
branch.isActive ? 1 : 0,
|
|
1825
|
-
branch.createdAt
|
|
1826
|
-
);
|
|
1827
|
-
}
|
|
1828
|
-
async getBranch(chatId, name) {
|
|
1829
|
-
const row = this.#db.prepare("SELECT * FROM branches WHERE chatId = ? AND name = ?").get(chatId, name);
|
|
1830
|
-
if (!row) {
|
|
1831
|
-
return void 0;
|
|
1832
|
-
}
|
|
1833
|
-
return {
|
|
1834
|
-
id: row.id,
|
|
1835
|
-
chatId: row.chatId,
|
|
1836
|
-
name: row.name,
|
|
1837
|
-
headMessageId: row.headMessageId,
|
|
1838
|
-
isActive: row.isActive === 1,
|
|
1839
|
-
createdAt: row.createdAt
|
|
1840
|
-
};
|
|
1841
|
-
}
|
|
1842
|
-
async getActiveBranch(chatId) {
|
|
1843
|
-
const row = this.#stmt(
|
|
1844
|
-
"SELECT * FROM branches WHERE chatId = ? AND isActive = 1"
|
|
1845
|
-
).get(chatId);
|
|
1846
|
-
if (!row) {
|
|
1847
|
-
return void 0;
|
|
1848
|
-
}
|
|
1849
|
-
return {
|
|
1850
|
-
id: row.id,
|
|
1851
|
-
chatId: row.chatId,
|
|
1852
|
-
name: row.name,
|
|
1853
|
-
headMessageId: row.headMessageId,
|
|
1854
|
-
isActive: true,
|
|
1855
|
-
createdAt: row.createdAt
|
|
1856
|
-
};
|
|
1857
|
-
}
|
|
1858
|
-
async setActiveBranch(chatId, branchId) {
|
|
1859
|
-
this.#db.prepare("UPDATE branches SET isActive = 0 WHERE chatId = ?").run(chatId);
|
|
1860
|
-
this.#db.prepare("UPDATE branches SET isActive = 1 WHERE id = ?").run(branchId);
|
|
1861
|
-
}
|
|
1862
|
-
async updateBranchHead(branchId, messageId) {
|
|
1863
|
-
this.#stmt("UPDATE branches SET headMessageId = ? WHERE id = ?").run(
|
|
1864
|
-
messageId,
|
|
1865
|
-
branchId
|
|
1866
|
-
);
|
|
1867
|
-
}
|
|
1868
|
-
async listBranches(chatId) {
|
|
1869
|
-
const rows = this.#db.prepare(
|
|
1870
|
-
`SELECT
|
|
1871
|
-
b.id,
|
|
1872
|
-
b.name,
|
|
1873
|
-
b.headMessageId,
|
|
1874
|
-
b.isActive,
|
|
1875
|
-
b.createdAt,
|
|
1876
|
-
COALESCE(
|
|
1877
|
-
(
|
|
1878
|
-
WITH RECURSIVE chain AS (
|
|
1879
|
-
SELECT id, parentId FROM messages WHERE id = b.headMessageId
|
|
1880
|
-
UNION ALL
|
|
1881
|
-
SELECT m.id, m.parentId FROM messages m
|
|
1882
|
-
INNER JOIN chain c ON m.id = c.parentId
|
|
1883
|
-
)
|
|
1884
|
-
SELECT COUNT(*) FROM chain
|
|
1885
|
-
),
|
|
1886
|
-
0
|
|
1887
|
-
) as messageCount
|
|
1888
|
-
FROM branches b
|
|
1889
|
-
WHERE b.chatId = ?
|
|
1890
|
-
ORDER BY b.createdAt ASC`
|
|
1891
|
-
).all(chatId);
|
|
1892
|
-
return rows.map((row) => ({
|
|
1893
|
-
id: row.id,
|
|
1894
|
-
name: row.name,
|
|
1895
|
-
headMessageId: row.headMessageId,
|
|
1896
|
-
isActive: row.isActive === 1,
|
|
1897
|
-
messageCount: row.messageCount,
|
|
1898
|
-
createdAt: row.createdAt
|
|
1899
|
-
}));
|
|
1900
|
-
}
|
|
1901
|
-
// ==========================================================================
|
|
1902
|
-
// Checkpoint Operations
|
|
1903
|
-
// ==========================================================================
|
|
1904
|
-
async createCheckpoint(checkpoint) {
|
|
1905
|
-
this.#db.prepare(
|
|
1906
|
-
`INSERT INTO checkpoints (id, chatId, name, messageId, createdAt)
|
|
1907
|
-
VALUES (?, ?, ?, ?, ?)
|
|
1908
|
-
ON CONFLICT(chatId, name) DO UPDATE SET
|
|
1909
|
-
messageId = excluded.messageId,
|
|
1910
|
-
createdAt = excluded.createdAt`
|
|
1911
|
-
).run(
|
|
1912
|
-
checkpoint.id,
|
|
1913
|
-
checkpoint.chatId,
|
|
1914
|
-
checkpoint.name,
|
|
1915
|
-
checkpoint.messageId,
|
|
1916
|
-
checkpoint.createdAt
|
|
1917
|
-
);
|
|
1918
|
-
}
|
|
1919
|
-
async getCheckpoint(chatId, name) {
|
|
1920
|
-
const row = this.#db.prepare("SELECT * FROM checkpoints WHERE chatId = ? AND name = ?").get(chatId, name);
|
|
1921
|
-
if (!row) {
|
|
1922
|
-
return void 0;
|
|
1923
|
-
}
|
|
1924
|
-
return {
|
|
1925
|
-
id: row.id,
|
|
1926
|
-
chatId: row.chatId,
|
|
1927
|
-
name: row.name,
|
|
1928
|
-
messageId: row.messageId,
|
|
1929
|
-
createdAt: row.createdAt
|
|
1930
|
-
};
|
|
1931
|
-
}
|
|
1932
|
-
async listCheckpoints(chatId) {
|
|
1933
|
-
const rows = this.#db.prepare(
|
|
1934
|
-
`SELECT id, name, messageId, createdAt
|
|
1935
|
-
FROM checkpoints
|
|
1936
|
-
WHERE chatId = ?
|
|
1937
|
-
ORDER BY createdAt DESC`
|
|
1938
|
-
).all(chatId);
|
|
1939
|
-
return rows.map((row) => ({
|
|
1940
|
-
id: row.id,
|
|
1941
|
-
name: row.name,
|
|
1942
|
-
messageId: row.messageId,
|
|
1943
|
-
createdAt: row.createdAt
|
|
1944
|
-
}));
|
|
1945
|
-
}
|
|
1946
|
-
async deleteCheckpoint(chatId, name) {
|
|
1947
|
-
this.#db.prepare("DELETE FROM checkpoints WHERE chatId = ? AND name = ?").run(chatId, name);
|
|
1948
|
-
}
|
|
1949
|
-
// ==========================================================================
|
|
1950
|
-
// Search Operations
|
|
1951
|
-
// ==========================================================================
|
|
1952
|
-
async searchMessages(chatId, query, options) {
|
|
1953
|
-
const limit = options?.limit ?? 20;
|
|
1954
|
-
const roles = options?.roles;
|
|
1955
|
-
let sql = `
|
|
1956
|
-
SELECT
|
|
1957
|
-
m.id,
|
|
1958
|
-
m.chatId,
|
|
1959
|
-
m.parentId,
|
|
1960
|
-
m.name,
|
|
1961
|
-
m.type,
|
|
1962
|
-
m.data,
|
|
1963
|
-
m.createdAt,
|
|
1964
|
-
fts.rank,
|
|
1965
|
-
snippet(messages_fts, 3, '<mark>', '</mark>', '...', 32) as snippet
|
|
1966
|
-
FROM messages_fts fts
|
|
1967
|
-
JOIN messages m ON m.id = fts.messageId
|
|
1968
|
-
WHERE messages_fts MATCH ?
|
|
1969
|
-
AND fts.chatId = ?
|
|
1970
|
-
`;
|
|
1971
|
-
const params = [query, chatId];
|
|
1972
|
-
if (roles && roles.length > 0) {
|
|
1973
|
-
const placeholders = roles.map(() => "?").join(", ");
|
|
1974
|
-
sql += ` AND fts.name IN (${placeholders})`;
|
|
1975
|
-
params.push(...roles);
|
|
1976
|
-
}
|
|
1977
|
-
sql += " ORDER BY fts.rank LIMIT ?";
|
|
1978
|
-
params.push(limit);
|
|
1979
|
-
const rows = this.#db.prepare(sql).all(...params);
|
|
1980
|
-
return rows.map((row) => ({
|
|
1981
|
-
message: {
|
|
1982
|
-
id: row.id,
|
|
1983
|
-
chatId: row.chatId,
|
|
1984
|
-
parentId: row.parentId,
|
|
1985
|
-
name: row.name,
|
|
1986
|
-
type: row.type ?? void 0,
|
|
1987
|
-
data: JSON.parse(row.data),
|
|
1988
|
-
createdAt: row.createdAt
|
|
1989
|
-
},
|
|
1990
|
-
rank: row.rank,
|
|
1991
|
-
snippet: row.snippet
|
|
1992
|
-
}));
|
|
1993
|
-
}
|
|
1994
|
-
// ==========================================================================
|
|
1995
|
-
// Visualization Operations
|
|
1996
|
-
// ==========================================================================
|
|
1997
|
-
async getGraph(chatId) {
|
|
1998
|
-
const messageRows = this.#db.prepare(
|
|
1999
|
-
`SELECT id, parentId, name, data, createdAt
|
|
2000
|
-
FROM messages
|
|
2001
|
-
WHERE chatId = ?
|
|
2002
|
-
ORDER BY createdAt ASC`
|
|
2003
|
-
).all(chatId);
|
|
2004
|
-
const nodes = messageRows.map((row) => {
|
|
2005
|
-
const data = JSON.parse(row.data);
|
|
2006
|
-
const content = typeof data === "string" ? data : JSON.stringify(data);
|
|
2007
|
-
return {
|
|
2008
|
-
id: row.id,
|
|
2009
|
-
parentId: row.parentId,
|
|
2010
|
-
role: row.name,
|
|
2011
|
-
content: content.length > 50 ? content.slice(0, 50) + "..." : content,
|
|
2012
|
-
createdAt: row.createdAt
|
|
2013
|
-
};
|
|
2014
|
-
});
|
|
2015
|
-
const branchRows = this.#db.prepare(
|
|
2016
|
-
`SELECT name, headMessageId, isActive
|
|
2017
|
-
FROM branches
|
|
2018
|
-
WHERE chatId = ?
|
|
2019
|
-
ORDER BY createdAt ASC`
|
|
2020
|
-
).all(chatId);
|
|
2021
|
-
const branches = branchRows.map((row) => ({
|
|
2022
|
-
name: row.name,
|
|
2023
|
-
headMessageId: row.headMessageId,
|
|
2024
|
-
isActive: row.isActive === 1
|
|
2025
|
-
}));
|
|
2026
|
-
const checkpointRows = this.#db.prepare(
|
|
2027
|
-
`SELECT name, messageId
|
|
2028
|
-
FROM checkpoints
|
|
2029
|
-
WHERE chatId = ?
|
|
2030
|
-
ORDER BY createdAt ASC`
|
|
2031
|
-
).all(chatId);
|
|
2032
|
-
const checkpoints = checkpointRows.map((row) => ({
|
|
2033
|
-
name: row.name,
|
|
2034
|
-
messageId: row.messageId
|
|
2035
|
-
}));
|
|
2036
|
-
return {
|
|
2037
|
-
chatId,
|
|
2038
|
-
nodes,
|
|
2039
|
-
branches,
|
|
2040
|
-
checkpoints
|
|
2041
|
-
};
|
|
2042
|
-
}
|
|
2043
|
-
};
|
|
2044
|
-
var InMemoryContextStore = class extends SqliteContextStore {
|
|
2045
|
-
constructor() {
|
|
2046
|
-
super(":memory:");
|
|
2047
|
-
}
|
|
2048
|
-
};
|
|
2049
|
-
function structuredOutput(options) {
|
|
2050
|
-
return {
|
|
2051
|
-
async generate(contextVariables, config) {
|
|
2052
|
-
if (!options.context) {
|
|
2053
|
-
throw new Error(`structuredOutput is missing a context.`);
|
|
2054
|
-
}
|
|
2055
|
-
if (!options.model) {
|
|
2056
|
-
throw new Error(`structuredOutput is missing a model.`);
|
|
2057
|
-
}
|
|
2058
|
-
const { messages, systemPrompt } = await options.context.resolve({
|
|
2059
|
-
renderer: new XmlRenderer()
|
|
2060
|
-
});
|
|
2061
|
-
const result = await generateText({
|
|
2062
|
-
abortSignal: config?.abortSignal,
|
|
2063
|
-
providerOptions: options.providerOptions,
|
|
2064
|
-
model: options.model,
|
|
2065
|
-
system: systemPrompt,
|
|
2066
|
-
messages: await convertToModelMessages(messages),
|
|
2067
|
-
stopWhen: stepCountIs(25),
|
|
2068
|
-
experimental_repairToolCall: repairToolCall,
|
|
2069
|
-
experimental_context: contextVariables,
|
|
2070
|
-
output: Output.object({ schema: options.schema }),
|
|
2071
|
-
tools: options.tools
|
|
2072
|
-
});
|
|
2073
|
-
return result.output;
|
|
2074
|
-
},
|
|
2075
|
-
async stream(contextVariables, config) {
|
|
2076
|
-
if (!options.context) {
|
|
2077
|
-
throw new Error(`structuredOutput is missing a context.`);
|
|
2078
|
-
}
|
|
2079
|
-
if (!options.model) {
|
|
2080
|
-
throw new Error(`structuredOutput is missing a model.`);
|
|
2081
|
-
}
|
|
2082
|
-
const { messages, systemPrompt } = await options.context.resolve({
|
|
2083
|
-
renderer: new XmlRenderer()
|
|
2084
|
-
});
|
|
2085
|
-
return streamText({
|
|
2086
|
-
abortSignal: config?.abortSignal,
|
|
2087
|
-
providerOptions: options.providerOptions,
|
|
2088
|
-
model: options.model,
|
|
2089
|
-
system: systemPrompt,
|
|
2090
|
-
experimental_repairToolCall: repairToolCall,
|
|
2091
|
-
messages: await convertToModelMessages(messages),
|
|
2092
|
-
stopWhen: stepCountIs(50),
|
|
2093
|
-
experimental_transform: config?.transform ?? smoothStream(),
|
|
2094
|
-
experimental_context: contextVariables,
|
|
2095
|
-
output: Output.object({ schema: options.schema }),
|
|
2096
|
-
tools: options.tools
|
|
2097
|
-
});
|
|
2098
|
-
}
|
|
2099
|
-
};
|
|
2100
|
-
}
|
|
2101
|
-
var repairToolCall = async ({
|
|
2102
|
-
toolCall,
|
|
2103
|
-
tools,
|
|
2104
|
-
inputSchema,
|
|
2105
|
-
error
|
|
2106
|
-
}) => {
|
|
2107
|
-
console.log(
|
|
2108
|
-
`Debug: ${chalk2.yellow("RepairingToolCall")}: ${toolCall.toolName}`,
|
|
2109
|
-
error.name
|
|
2110
|
-
);
|
|
2111
|
-
if (NoSuchToolError.isInstance(error)) {
|
|
2112
|
-
return null;
|
|
2113
|
-
}
|
|
2114
|
-
const tool = tools[toolCall.toolName];
|
|
2115
|
-
const { output } = await generateText({
|
|
2116
|
-
model: groq("openai/gpt-oss-20b"),
|
|
2117
|
-
output: Output.object({ schema: tool.inputSchema }),
|
|
2118
|
-
prompt: [
|
|
2119
|
-
`The model tried to call the tool "${toolCall.toolName}" with the following inputs:`,
|
|
2120
|
-
JSON.stringify(toolCall.input),
|
|
2121
|
-
`The tool accepts the following schema:`,
|
|
2122
|
-
JSON.stringify(inputSchema(toolCall)),
|
|
2123
|
-
"Please fix the inputs."
|
|
2124
|
-
].join("\n")
|
|
2125
|
-
});
|
|
2126
|
-
return { ...toolCall, input: JSON.stringify(output) };
|
|
2127
|
-
};
|
|
2128
|
-
|
|
2129
|
-
// packages/text2sql/src/lib/synthesis/extractors/base-contextual-extractor.ts
|
|
168
|
+
import { groq } from "@ai-sdk/groq";
|
|
169
|
+
import {
|
|
170
|
+
getToolOrDynamicToolName,
|
|
171
|
+
isTextUIPart,
|
|
172
|
+
isToolOrDynamicToolUIPart
|
|
173
|
+
} from "ai";
|
|
174
|
+
import dedent from "dedent";
|
|
175
|
+
import z from "zod";
|
|
176
|
+
import {
|
|
177
|
+
ContextEngine,
|
|
178
|
+
InMemoryContextStore,
|
|
179
|
+
fragment,
|
|
180
|
+
persona,
|
|
181
|
+
structuredOutput,
|
|
182
|
+
user
|
|
183
|
+
} from "@deepagents/context";
|
|
2130
184
|
var contextResolverSchema = z.object({
|
|
2131
185
|
question: z.string().describe(
|
|
2132
186
|
"A standalone natural language question that the SQL query answers"
|
|
@@ -2149,7 +203,7 @@ async function resolveContext(params) {
|
|
|
2149
203
|
fragment("sql", params.sql),
|
|
2150
204
|
fragment(
|
|
2151
205
|
"task",
|
|
2152
|
-
|
|
206
|
+
dedent`
|
|
2153
207
|
Given the conversation above and the SQL query that was executed,
|
|
2154
208
|
generate a single, standalone natural language question that:
|
|
2155
209
|
1. Fully captures the user's intent without needing prior context
|
|
@@ -2160,7 +214,7 @@ async function resolveContext(params) {
|
|
|
2160
214
|
),
|
|
2161
215
|
fragment(
|
|
2162
216
|
"examples",
|
|
2163
|
-
|
|
217
|
+
dedent`
|
|
2164
218
|
Conversation: "Show me customers" → "Filter to NY" → "Sort by revenue"
|
|
2165
219
|
SQL: SELECT * FROM customers WHERE region = 'NY' ORDER BY revenue DESC
|
|
2166
220
|
Question: "Show me customers in the NY region sorted by revenue"
|
|
@@ -2173,14 +227,14 @@ async function resolveContext(params) {
|
|
|
2173
227
|
user("Generate a standalone question for this SQL query.")
|
|
2174
228
|
);
|
|
2175
229
|
const resolverOutput = structuredOutput({
|
|
2176
|
-
model:
|
|
230
|
+
model: groq("openai/gpt-oss-20b"),
|
|
2177
231
|
context,
|
|
2178
232
|
schema: contextResolverSchema
|
|
2179
233
|
});
|
|
2180
234
|
return resolverOutput.generate();
|
|
2181
235
|
}
|
|
2182
|
-
function getMessageText(
|
|
2183
|
-
const textParts =
|
|
236
|
+
function getMessageText(message) {
|
|
237
|
+
const textParts = message.parts.filter(isTextUIPart).map((part) => part.text);
|
|
2184
238
|
return textParts.join(" ").trim();
|
|
2185
239
|
}
|
|
2186
240
|
function formatConversation(messages) {
|
|
@@ -2217,24 +271,24 @@ var BaseContextualExtractor = class extends PairProducer {
|
|
|
2217
271
|
* Core extraction loop - iterates through messages and calls hooks.
|
|
2218
272
|
*/
|
|
2219
273
|
async extractSqlsWithContext(toolName, includeFailures) {
|
|
2220
|
-
for (const
|
|
2221
|
-
if (
|
|
2222
|
-
const text = getMessageText(
|
|
274
|
+
for (const message of this.messages) {
|
|
275
|
+
if (message.role === "user") {
|
|
276
|
+
const text = getMessageText(message);
|
|
2223
277
|
if (text) {
|
|
2224
278
|
await this.onUserMessage(text);
|
|
2225
279
|
}
|
|
2226
280
|
continue;
|
|
2227
281
|
}
|
|
2228
|
-
if (
|
|
2229
|
-
await this.extractFromAssistant(
|
|
282
|
+
if (message.role === "assistant") {
|
|
283
|
+
await this.extractFromAssistant(message, toolName, includeFailures);
|
|
2230
284
|
}
|
|
2231
285
|
}
|
|
2232
286
|
}
|
|
2233
287
|
/**
|
|
2234
288
|
* Extract SQL from assistant message parts.
|
|
2235
289
|
*/
|
|
2236
|
-
async extractFromAssistant(
|
|
2237
|
-
for (const part of
|
|
290
|
+
async extractFromAssistant(message, toolName, includeFailures) {
|
|
291
|
+
for (const part of message.parts) {
|
|
2238
292
|
if (!isToolOrDynamicToolUIPart(part)) {
|
|
2239
293
|
continue;
|
|
2240
294
|
}
|
|
@@ -2263,9 +317,9 @@ var BaseContextualExtractor = class extends PairProducer {
|
|
|
2263
317
|
conversationContext: snapshot
|
|
2264
318
|
});
|
|
2265
319
|
}
|
|
2266
|
-
const
|
|
2267
|
-
if (
|
|
2268
|
-
this.context.push(`Assistant: ${
|
|
320
|
+
const assistantText = getMessageText(message);
|
|
321
|
+
if (assistantText) {
|
|
322
|
+
this.context.push(`Assistant: ${assistantText}`);
|
|
2269
323
|
}
|
|
2270
324
|
}
|
|
2271
325
|
/**
|
|
@@ -2310,13 +364,13 @@ var MessageExtractor = class extends PairProducer {
|
|
|
2310
364
|
async *produce() {
|
|
2311
365
|
const { includeFailures = false, toolName = "db_query" } = this.#options;
|
|
2312
366
|
let lastUserMessage = null;
|
|
2313
|
-
for (const
|
|
2314
|
-
if (
|
|
2315
|
-
lastUserMessage =
|
|
367
|
+
for (const message of this.#messages) {
|
|
368
|
+
if (message.role === "user") {
|
|
369
|
+
lastUserMessage = message;
|
|
2316
370
|
continue;
|
|
2317
371
|
}
|
|
2318
|
-
if (
|
|
2319
|
-
for (const part of
|
|
372
|
+
if (message.role === "assistant" && lastUserMessage) {
|
|
373
|
+
for (const part of message.parts) {
|
|
2320
374
|
if (!isToolOrDynamicToolUIPart2(part)) {
|
|
2321
375
|
continue;
|
|
2322
376
|
}
|
|
@@ -2353,9 +407,17 @@ var MessageExtractor = class extends PairProducer {
|
|
|
2353
407
|
};
|
|
2354
408
|
|
|
2355
409
|
// packages/text2sql/src/lib/synthesis/extractors/sql-extractor.ts
|
|
2356
|
-
import { groq as
|
|
2357
|
-
import
|
|
410
|
+
import { groq as groq2 } from "@ai-sdk/groq";
|
|
411
|
+
import dedent2 from "dedent";
|
|
2358
412
|
import z2 from "zod";
|
|
413
|
+
import {
|
|
414
|
+
ContextEngine as ContextEngine2,
|
|
415
|
+
InMemoryContextStore as InMemoryContextStore2,
|
|
416
|
+
fragment as fragment2,
|
|
417
|
+
persona as persona2,
|
|
418
|
+
structuredOutput as structuredOutput2,
|
|
419
|
+
user as user2
|
|
420
|
+
} from "@deepagents/context";
|
|
2359
421
|
var outputSchema = z2.object({
|
|
2360
422
|
question: z2.string().describe("A natural language question that the SQL query answers")
|
|
2361
423
|
});
|
|
@@ -2390,22 +452,22 @@ var SqlExtractor = class extends PairProducer {
|
|
|
2390
452
|
continue;
|
|
2391
453
|
}
|
|
2392
454
|
}
|
|
2393
|
-
const context = new
|
|
2394
|
-
store: new
|
|
455
|
+
const context = new ContextEngine2({
|
|
456
|
+
store: new InMemoryContextStore2(),
|
|
2395
457
|
chatId: `sql-to-question-${crypto.randomUUID()}`,
|
|
2396
458
|
userId: "system"
|
|
2397
459
|
});
|
|
2398
460
|
context.set(
|
|
2399
|
-
|
|
461
|
+
persona2({
|
|
2400
462
|
name: "sql_to_question",
|
|
2401
463
|
role: "You are an expert at understanding SQL queries and generating clear, natural language questions that describe what the query retrieves.",
|
|
2402
464
|
objective: "Generate clear, natural language questions that describe what SQL queries retrieve"
|
|
2403
465
|
}),
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
466
|
+
fragment2("database_schema", introspection),
|
|
467
|
+
fragment2("sql", sql),
|
|
468
|
+
fragment2(
|
|
2407
469
|
"task",
|
|
2408
|
-
|
|
470
|
+
dedent2`
|
|
2409
471
|
Given the database schema and the SQL query above, generate a single
|
|
2410
472
|
natural language question that:
|
|
2411
473
|
1. Accurately describes what information the query retrieves
|
|
@@ -2414,9 +476,9 @@ var SqlExtractor = class extends PairProducer {
|
|
|
2414
476
|
4. Is concise but complete
|
|
2415
477
|
`
|
|
2416
478
|
),
|
|
2417
|
-
|
|
479
|
+
fragment2(
|
|
2418
480
|
"examples",
|
|
2419
|
-
|
|
481
|
+
dedent2`
|
|
2420
482
|
SQL: SELECT COUNT(*) FROM customers WHERE region = 'NY'
|
|
2421
483
|
Question: "How many customers do we have in New York?"
|
|
2422
484
|
|
|
@@ -2427,10 +489,10 @@ var SqlExtractor = class extends PairProducer {
|
|
|
2427
489
|
Question: "Which customers have never placed an order?"
|
|
2428
490
|
`
|
|
2429
491
|
),
|
|
2430
|
-
|
|
492
|
+
user2("Generate a natural language question for this SQL query.")
|
|
2431
493
|
);
|
|
2432
|
-
const sqlToQuestionOutput =
|
|
2433
|
-
model:
|
|
494
|
+
const sqlToQuestionOutput = structuredOutput2({
|
|
495
|
+
model: groq2("openai/gpt-oss-20b"),
|
|
2434
496
|
context,
|
|
2435
497
|
schema: outputSchema
|
|
2436
498
|
});
|
|
@@ -2490,30 +552,38 @@ var WindowedContextExtractor = class extends BaseContextualExtractor {
|
|
|
2490
552
|
};
|
|
2491
553
|
|
|
2492
554
|
// packages/text2sql/src/lib/synthesis/extractors/segmented-context-extractor.ts
|
|
2493
|
-
import { groq as
|
|
2494
|
-
import
|
|
555
|
+
import { groq as groq3 } from "@ai-sdk/groq";
|
|
556
|
+
import dedent3 from "dedent";
|
|
2495
557
|
import z3 from "zod";
|
|
558
|
+
import {
|
|
559
|
+
ContextEngine as ContextEngine3,
|
|
560
|
+
InMemoryContextStore as InMemoryContextStore3,
|
|
561
|
+
fragment as fragment3,
|
|
562
|
+
persona as persona3,
|
|
563
|
+
structuredOutput as structuredOutput3,
|
|
564
|
+
user as user3
|
|
565
|
+
} from "@deepagents/context";
|
|
2496
566
|
var topicChangeSchema = z3.object({
|
|
2497
567
|
isTopicChange: z3.boolean().describe("Whether the new message represents a topic change"),
|
|
2498
568
|
reason: z3.string().describe("Brief explanation for the decision")
|
|
2499
569
|
});
|
|
2500
570
|
async function detectTopicChange(params) {
|
|
2501
|
-
const context = new
|
|
2502
|
-
store: new
|
|
571
|
+
const context = new ContextEngine3({
|
|
572
|
+
store: new InMemoryContextStore3(),
|
|
2503
573
|
chatId: `topic-change-${crypto.randomUUID()}`,
|
|
2504
574
|
userId: "system"
|
|
2505
575
|
});
|
|
2506
576
|
context.set(
|
|
2507
|
-
|
|
577
|
+
persona3({
|
|
2508
578
|
name: "topic_change_detector",
|
|
2509
579
|
role: "You are an expert at understanding conversational flow and detecting topic changes.",
|
|
2510
580
|
objective: "Detect significant topic changes in database conversations"
|
|
2511
581
|
}),
|
|
2512
|
-
|
|
2513
|
-
|
|
2514
|
-
|
|
582
|
+
fragment3("conversation_context", params.context || "(no prior context)"),
|
|
583
|
+
fragment3("new_message", params.newMessage),
|
|
584
|
+
fragment3(
|
|
2515
585
|
"task",
|
|
2516
|
-
|
|
586
|
+
dedent3`
|
|
2517
587
|
Determine if the new message represents a significant topic change from the
|
|
2518
588
|
prior conversation context. A topic change occurs when:
|
|
2519
589
|
1. The user asks about a completely different entity/table/domain
|
|
@@ -2526,9 +596,9 @@ async function detectTopicChange(params) {
|
|
|
2526
596
|
- Requests for more details on the same topic
|
|
2527
597
|
`
|
|
2528
598
|
),
|
|
2529
|
-
|
|
599
|
+
fragment3(
|
|
2530
600
|
"examples",
|
|
2531
|
-
|
|
601
|
+
dedent3`
|
|
2532
602
|
Context: "Show me customers in NY" → "Sort by revenue"
|
|
2533
603
|
New: "Filter to those with orders over $1000"
|
|
2534
604
|
Decision: NOT a topic change (still refining customer query)
|
|
@@ -2542,10 +612,10 @@ async function detectTopicChange(params) {
|
|
|
2542
612
|
Decision: Topic change (products → orders/sales)
|
|
2543
613
|
`
|
|
2544
614
|
),
|
|
2545
|
-
|
|
615
|
+
user3("Determine if this is a topic change.")
|
|
2546
616
|
);
|
|
2547
|
-
const topicOutput =
|
|
2548
|
-
model:
|
|
617
|
+
const topicOutput = structuredOutput3({
|
|
618
|
+
model: groq3("openai/gpt-oss-20b"),
|
|
2549
619
|
context,
|
|
2550
620
|
schema: topicChangeSchema
|
|
2551
621
|
});
|
|
@@ -2645,12 +715,21 @@ var LastQueryExtractor = class extends BaseContextualExtractor {
|
|
|
2645
715
|
import pLimit from "p-limit";
|
|
2646
716
|
|
|
2647
717
|
// packages/text2sql/src/lib/agents/question.agent.ts
|
|
2648
|
-
import { groq as
|
|
2649
|
-
import
|
|
718
|
+
import { groq as groq4 } from "@ai-sdk/groq";
|
|
719
|
+
import dedent4 from "dedent";
|
|
2650
720
|
import z4 from "zod";
|
|
2651
721
|
import "@deepagents/agent";
|
|
722
|
+
import {
|
|
723
|
+
ContextEngine as ContextEngine4,
|
|
724
|
+
InMemoryContextStore as InMemoryContextStore4,
|
|
725
|
+
fragment as fragment4,
|
|
726
|
+
guardrail,
|
|
727
|
+
persona as persona4,
|
|
728
|
+
structuredOutput as structuredOutput4,
|
|
729
|
+
user as user4
|
|
730
|
+
} from "@deepagents/context";
|
|
2652
731
|
var complexityInstructions = {
|
|
2653
|
-
simple:
|
|
732
|
+
simple: dedent4`
|
|
2654
733
|
Generate simple questions that require:
|
|
2655
734
|
- Basic SELECT with single table
|
|
2656
735
|
- Simple WHERE clauses with one condition
|
|
@@ -2658,7 +737,7 @@ var complexityInstructions = {
|
|
|
2658
737
|
- No joins required
|
|
2659
738
|
Examples: "How many customers do we have?", "List all products", "What is the total revenue?"
|
|
2660
739
|
`,
|
|
2661
|
-
moderate:
|
|
740
|
+
moderate: dedent4`
|
|
2662
741
|
Generate moderate questions that require:
|
|
2663
742
|
- JOINs between 2-3 tables
|
|
2664
743
|
- Multiple WHERE conditions (AND/OR)
|
|
@@ -2667,7 +746,7 @@ var complexityInstructions = {
|
|
|
2667
746
|
- Basic subqueries
|
|
2668
747
|
Examples: "What are the top 5 customers by total orders?", "Which products have never been ordered?"
|
|
2669
748
|
`,
|
|
2670
|
-
complex:
|
|
749
|
+
complex: dedent4`
|
|
2671
750
|
Generate complex questions that require:
|
|
2672
751
|
- Multiple JOINs (3+ tables)
|
|
2673
752
|
- Nested subqueries or CTEs
|
|
@@ -2676,7 +755,7 @@ var complexityInstructions = {
|
|
|
2676
755
|
- Date/time calculations
|
|
2677
756
|
Examples: "What is the month-over-month growth rate?", "Which customers have increased spending compared to last year?"
|
|
2678
757
|
`,
|
|
2679
|
-
"high complex":
|
|
758
|
+
"high complex": dedent4`
|
|
2680
759
|
Generate highly complex questions that require advanced SQL features:
|
|
2681
760
|
- Window functions (ROW_NUMBER, RANK, DENSE_RANK)
|
|
2682
761
|
- LAG, LEAD for comparisons
|
|
@@ -2692,26 +771,26 @@ var outputSchema2 = z4.object({
|
|
|
2692
771
|
});
|
|
2693
772
|
async function generateQuestions(params) {
|
|
2694
773
|
const { introspection, complexity, count, prompt, model } = params;
|
|
2695
|
-
const context = new
|
|
2696
|
-
store: new
|
|
774
|
+
const context = new ContextEngine4({
|
|
775
|
+
store: new InMemoryContextStore4(),
|
|
2697
776
|
chatId: `question-gen-${crypto.randomUUID()}`,
|
|
2698
777
|
userId: "system"
|
|
2699
778
|
});
|
|
2700
779
|
context.set(
|
|
2701
|
-
|
|
780
|
+
persona4({
|
|
2702
781
|
name: "question_generator",
|
|
2703
782
|
role: "You are a synthetic data generator specializing in creating realistic natural language questions that users might ask about a database.",
|
|
2704
783
|
objective: "Generate diverse, realistic natural language questions that match the specified complexity level"
|
|
2705
784
|
}),
|
|
2706
|
-
|
|
2707
|
-
|
|
785
|
+
fragment4("database_schema", introspection || ""),
|
|
786
|
+
fragment4(
|
|
2708
787
|
"complexity",
|
|
2709
788
|
{ level: complexity },
|
|
2710
789
|
complexityInstructions[complexity]
|
|
2711
790
|
),
|
|
2712
|
-
|
|
791
|
+
fragment4(
|
|
2713
792
|
"task",
|
|
2714
|
-
|
|
793
|
+
dedent4`
|
|
2715
794
|
Generate exactly ${count} natural language questions at the "${complexity}" complexity level.
|
|
2716
795
|
The questions should:
|
|
2717
796
|
1. Match the complexity requirements above
|
|
@@ -2735,12 +814,12 @@ async function generateQuestions(params) {
|
|
|
2735
814
|
guardrail({
|
|
2736
815
|
rule: "All questions must match the specified complexity level"
|
|
2737
816
|
}),
|
|
2738
|
-
|
|
817
|
+
user4(
|
|
2739
818
|
prompt ?? `Generate ${count} questions at ${complexity} complexity given db schema.`
|
|
2740
819
|
)
|
|
2741
820
|
);
|
|
2742
|
-
const questionOutput =
|
|
2743
|
-
model: model ??
|
|
821
|
+
const questionOutput = structuredOutput4({
|
|
822
|
+
model: model ?? groq4("openai/gpt-oss-20b"),
|
|
2744
823
|
context,
|
|
2745
824
|
schema: outputSchema2
|
|
2746
825
|
});
|
|
@@ -2748,7 +827,7 @@ async function generateQuestions(params) {
|
|
|
2748
827
|
}
|
|
2749
828
|
|
|
2750
829
|
// packages/text2sql/src/lib/agents/sql.agent.ts
|
|
2751
|
-
import { groq as
|
|
830
|
+
import { groq as groq5 } from "@ai-sdk/groq";
|
|
2752
831
|
import {
|
|
2753
832
|
APICallError,
|
|
2754
833
|
JSONParseError,
|
|
@@ -2764,6 +843,13 @@ import { createWriteStream } from "node:fs";
|
|
|
2764
843
|
import pRetry from "p-retry";
|
|
2765
844
|
import z5 from "zod";
|
|
2766
845
|
import "@deepagents/agent";
|
|
846
|
+
import {
|
|
847
|
+
ContextEngine as ContextEngine5,
|
|
848
|
+
InMemoryContextStore as InMemoryContextStore5,
|
|
849
|
+
persona as persona5,
|
|
850
|
+
structuredOutput as structuredOutput5,
|
|
851
|
+
user as user5
|
|
852
|
+
} from "@deepagents/context";
|
|
2767
853
|
var logger = new Console({
|
|
2768
854
|
stdout: createWriteStream("./sql-agent.log", { flags: "a" }),
|
|
2769
855
|
stderr: createWriteStream("./sql-agent-error.log", { flags: "a" }),
|
|
@@ -2777,8 +863,8 @@ function extractSql(output) {
|
|
|
2777
863
|
var marker = Symbol("SQLValidationError");
|
|
2778
864
|
var SQLValidationError = class _SQLValidationError extends Error {
|
|
2779
865
|
[marker];
|
|
2780
|
-
constructor(
|
|
2781
|
-
super(
|
|
866
|
+
constructor(message) {
|
|
867
|
+
super(message);
|
|
2782
868
|
this.name = "SQLValidationError";
|
|
2783
869
|
this[marker] = true;
|
|
2784
870
|
}
|
|
@@ -2787,8 +873,8 @@ var SQLValidationError = class _SQLValidationError extends Error {
|
|
|
2787
873
|
}
|
|
2788
874
|
};
|
|
2789
875
|
var UnanswerableSQLError = class _UnanswerableSQLError extends Error {
|
|
2790
|
-
constructor(
|
|
2791
|
-
super(
|
|
876
|
+
constructor(message) {
|
|
877
|
+
super(message);
|
|
2792
878
|
this.name = "UnanswerableSQLError";
|
|
2793
879
|
}
|
|
2794
880
|
static isInstance(error) {
|
|
@@ -2799,13 +885,13 @@ async function toSql(options) {
|
|
|
2799
885
|
const { maxRetries = 3 } = options;
|
|
2800
886
|
return withRetry(
|
|
2801
887
|
async (attemptNumber, errors, attempts) => {
|
|
2802
|
-
const context = new
|
|
2803
|
-
store: new
|
|
888
|
+
const context = new ContextEngine5({
|
|
889
|
+
store: new InMemoryContextStore5(),
|
|
2804
890
|
chatId: `sql-gen-${crypto.randomUUID()}`,
|
|
2805
891
|
userId: "system"
|
|
2806
892
|
});
|
|
2807
893
|
context.set(
|
|
2808
|
-
|
|
894
|
+
persona5({
|
|
2809
895
|
name: "Freya",
|
|
2810
896
|
role: "You are an expert SQL query generator. You translate natural language questions into precise, efficient SQL queries based on the provided database schema.",
|
|
2811
897
|
objective: "Translate natural language questions into precise, efficient SQL queries"
|
|
@@ -2815,21 +901,21 @@ async function toSql(options) {
|
|
|
2815
901
|
);
|
|
2816
902
|
if (errors.length) {
|
|
2817
903
|
context.set(
|
|
2818
|
-
|
|
2819
|
-
|
|
904
|
+
user5(options.input),
|
|
905
|
+
user5(
|
|
2820
906
|
`<validation_error>Your previous SQL query had the following error: ${errors.at(-1)?.message}. Please fix the query.</validation_error>`
|
|
2821
907
|
)
|
|
2822
908
|
);
|
|
2823
909
|
} else {
|
|
2824
|
-
context.set(
|
|
910
|
+
context.set(user5(options.input));
|
|
2825
911
|
}
|
|
2826
912
|
const temperature = RETRY_TEMPERATURES[attemptNumber - 1] ?? RETRY_TEMPERATURES[RETRY_TEMPERATURES.length - 1];
|
|
2827
|
-
const baseModel = options.model ??
|
|
913
|
+
const baseModel = options.model ?? groq5("openai/gpt-oss-20b");
|
|
2828
914
|
const model = wrapLanguageModel({
|
|
2829
915
|
model: baseModel,
|
|
2830
916
|
middleware: defaultSettingsMiddleware({ settings: { temperature } })
|
|
2831
917
|
});
|
|
2832
|
-
const sqlOutput =
|
|
918
|
+
const sqlOutput = structuredOutput5({
|
|
2833
919
|
model,
|
|
2834
920
|
context,
|
|
2835
921
|
schema: z5.union([
|
|
@@ -2943,12 +1029,12 @@ var SchemaSynthesizer = class extends PairProducer {
|
|
|
2943
1029
|
async *produce() {
|
|
2944
1030
|
const introspection = "";
|
|
2945
1031
|
const combinations = this.#personas.flatMap(
|
|
2946
|
-
(
|
|
1032
|
+
(persona8) => this.#complexities.map((complexity) => ({ persona: persona8, complexity }))
|
|
2947
1033
|
);
|
|
2948
|
-
for (const { persona:
|
|
1034
|
+
for (const { persona: persona8, complexity } of combinations) {
|
|
2949
1035
|
const pairs = await this.#processCombination(
|
|
2950
1036
|
introspection,
|
|
2951
|
-
|
|
1037
|
+
persona8,
|
|
2952
1038
|
complexity
|
|
2953
1039
|
);
|
|
2954
1040
|
if (pairs.length) {
|
|
@@ -2960,8 +1046,8 @@ var SchemaSynthesizer = class extends PairProducer {
|
|
|
2960
1046
|
* Processes a single persona × complexity combination by generating questions
|
|
2961
1047
|
* and converting each to SQL in parallel.
|
|
2962
1048
|
*/
|
|
2963
|
-
async #processCombination(introspection,
|
|
2964
|
-
const personaContext =
|
|
1049
|
+
async #processCombination(introspection, persona8, complexity) {
|
|
1050
|
+
const personaContext = persona8 ? `As ${persona8.role}, ${persona8.perspective}
|
|
2965
1051
|
|
|
2966
1052
|
Generate questions this persona would ask.` : void 0;
|
|
2967
1053
|
const prompt = personaContext ? `${personaContext}
|
|
@@ -3013,11 +1099,20 @@ Generate ${this.options.count} questions at ${complexity} complexity.` : void 0;
|
|
|
3013
1099
|
};
|
|
3014
1100
|
|
|
3015
1101
|
// packages/text2sql/src/lib/synthesis/synthesizers/breadth-evolver.ts
|
|
3016
|
-
import { groq as
|
|
3017
|
-
import
|
|
1102
|
+
import { groq as groq6 } from "@ai-sdk/groq";
|
|
1103
|
+
import dedent5 from "dedent";
|
|
3018
1104
|
import pLimit2 from "p-limit";
|
|
3019
1105
|
import z6 from "zod";
|
|
3020
1106
|
import "@deepagents/agent";
|
|
1107
|
+
import {
|
|
1108
|
+
ContextEngine as ContextEngine6,
|
|
1109
|
+
InMemoryContextStore as InMemoryContextStore6,
|
|
1110
|
+
fragment as fragment5,
|
|
1111
|
+
guardrail as guardrail2,
|
|
1112
|
+
persona as personaFragment,
|
|
1113
|
+
structuredOutput as structuredOutput6,
|
|
1114
|
+
user as user6
|
|
1115
|
+
} from "@deepagents/context";
|
|
3021
1116
|
|
|
3022
1117
|
// packages/text2sql/src/lib/synthesis/synthesizers/styles.ts
|
|
3023
1118
|
var ALL_STYLES = [
|
|
@@ -3059,12 +1154,12 @@ var paraphraserOutputSchema = z6.object({
|
|
|
3059
1154
|
).min(1).describe("List of paraphrased questions that would produce the same SQL")
|
|
3060
1155
|
});
|
|
3061
1156
|
async function paraphraseQuestion(params) {
|
|
3062
|
-
const context = new
|
|
3063
|
-
store: new
|
|
1157
|
+
const context = new ContextEngine6({
|
|
1158
|
+
store: new InMemoryContextStore6(),
|
|
3064
1159
|
chatId: `paraphraser-${crypto.randomUUID()}`,
|
|
3065
1160
|
userId: "system"
|
|
3066
1161
|
});
|
|
3067
|
-
const personaInstruction = params.persona ?
|
|
1162
|
+
const personaInstruction = params.persona ? dedent5`
|
|
3068
1163
|
<persona role="${params.persona.role}">
|
|
3069
1164
|
${params.persona.perspective}
|
|
3070
1165
|
|
|
@@ -3072,7 +1167,7 @@ async function paraphraseQuestion(params) {
|
|
|
3072
1167
|
Use their vocabulary, priorities, and framing style.
|
|
3073
1168
|
</persona>
|
|
3074
1169
|
` : "";
|
|
3075
|
-
const styleInstruction = params.persona?.styles && params.persona.styles.length > 0 ?
|
|
1170
|
+
const styleInstruction = params.persona?.styles && params.persona.styles.length > 0 ? dedent5`
|
|
3076
1171
|
<communication_styles>
|
|
3077
1172
|
Generate paraphrases using these communication styles: ${params.persona.styles.join(", ")}
|
|
3078
1173
|
|
|
@@ -3083,22 +1178,22 @@ async function paraphraseQuestion(params) {
|
|
|
3083
1178
|
</communication_styles>
|
|
3084
1179
|
` : "";
|
|
3085
1180
|
context.set(
|
|
3086
|
-
|
|
1181
|
+
personaFragment({
|
|
3087
1182
|
name: "question_paraphraser",
|
|
3088
1183
|
role: "You are a linguistic expert specializing in paraphrasing database questions. Your task is to generate alternative phrasings of questions that preserve the exact same semantic meaning - they must all produce the identical SQL query.",
|
|
3089
1184
|
objective: "Generate paraphrased versions of questions that preserve exact semantic meaning and produce identical SQL"
|
|
3090
1185
|
}),
|
|
3091
|
-
|
|
3092
|
-
|
|
1186
|
+
fragment5("original_question", params.question),
|
|
1187
|
+
fragment5(
|
|
3093
1188
|
"reference_sql",
|
|
3094
1189
|
params.sql,
|
|
3095
1190
|
"This SQL shows what the question is really asking - all paraphrases must ask for exactly this"
|
|
3096
1191
|
),
|
|
3097
|
-
...personaInstruction ? [
|
|
3098
|
-
...styleInstruction ? [
|
|
3099
|
-
|
|
1192
|
+
...personaInstruction ? [fragment5("persona", personaInstruction)] : [],
|
|
1193
|
+
...styleInstruction ? [fragment5("communication_styles", styleInstruction)] : [],
|
|
1194
|
+
fragment5(
|
|
3100
1195
|
"task",
|
|
3101
|
-
|
|
1196
|
+
dedent5`
|
|
3102
1197
|
Generate exactly ${params.count} paraphrased versions of the original question.
|
|
3103
1198
|
|
|
3104
1199
|
Requirements:
|
|
@@ -3110,22 +1205,22 @@ async function paraphraseQuestion(params) {
|
|
|
3110
1205
|
${params.persona?.styles?.length ? "6. Apply the specified communication styles to create diverse phrasings" : ""}
|
|
3111
1206
|
`
|
|
3112
1207
|
),
|
|
3113
|
-
|
|
3114
|
-
|
|
1208
|
+
guardrail2({ rule: "NEVER change what data is being requested" }),
|
|
1209
|
+
guardrail2({
|
|
3115
1210
|
rule: "NEVER add filters, aggregations, or conditions not in the original"
|
|
3116
1211
|
}),
|
|
3117
|
-
|
|
1212
|
+
guardrail2({
|
|
3118
1213
|
rule: "NEVER remove any specificity from the original question"
|
|
3119
1214
|
}),
|
|
3120
|
-
|
|
1215
|
+
guardrail2({
|
|
3121
1216
|
rule: "All paraphrases must be answerable by the exact same SQL query"
|
|
3122
1217
|
}),
|
|
3123
|
-
|
|
1218
|
+
user6(
|
|
3124
1219
|
`Paraphrase this question ${params.count} times: "${params.question}"`
|
|
3125
1220
|
)
|
|
3126
1221
|
);
|
|
3127
|
-
const paraphraserOutput =
|
|
3128
|
-
model: params.model ??
|
|
1222
|
+
const paraphraserOutput = structuredOutput6({
|
|
1223
|
+
model: params.model ?? groq6("openai/gpt-oss-20b"),
|
|
3129
1224
|
context,
|
|
3130
1225
|
schema: paraphraserOutputSchema
|
|
3131
1226
|
});
|
|
@@ -3173,15 +1268,24 @@ var BreadthEvolver = class extends PairProducer {
|
|
|
3173
1268
|
};
|
|
3174
1269
|
|
|
3175
1270
|
// packages/text2sql/src/lib/synthesis/synthesizers/depth-evolver.ts
|
|
3176
|
-
import { groq as
|
|
1271
|
+
import { groq as groq7 } from "@ai-sdk/groq";
|
|
3177
1272
|
import { NoObjectGeneratedError as NoObjectGeneratedError2, NoOutputGeneratedError as NoOutputGeneratedError2 } from "ai";
|
|
3178
|
-
import
|
|
1273
|
+
import dedent6 from "dedent";
|
|
3179
1274
|
import pLimit3 from "p-limit";
|
|
3180
1275
|
import pRetry2 from "p-retry";
|
|
3181
1276
|
import z7 from "zod";
|
|
3182
1277
|
import "@deepagents/agent";
|
|
1278
|
+
import {
|
|
1279
|
+
ContextEngine as ContextEngine7,
|
|
1280
|
+
InMemoryContextStore as InMemoryContextStore7,
|
|
1281
|
+
fragment as fragment6,
|
|
1282
|
+
guardrail as guardrail3,
|
|
1283
|
+
persona as persona6,
|
|
1284
|
+
structuredOutput as structuredOutput7,
|
|
1285
|
+
user as user7
|
|
1286
|
+
} from "@deepagents/context";
|
|
3183
1287
|
var techniqueInstructions = {
|
|
3184
|
-
"add-aggregation":
|
|
1288
|
+
"add-aggregation": dedent6`
|
|
3185
1289
|
Add aggregation requirements to the question.
|
|
3186
1290
|
Transform it to require GROUP BY, COUNT, SUM, AVG, MIN, MAX, or similar operations.
|
|
3187
1291
|
Examples:
|
|
@@ -3189,7 +1293,7 @@ var techniqueInstructions = {
|
|
|
3189
1293
|
- "List products" → "What is the average price per category?"
|
|
3190
1294
|
- "Get employees" → "How many employees are in each department?"
|
|
3191
1295
|
`,
|
|
3192
|
-
"add-filter":
|
|
1296
|
+
"add-filter": dedent6`
|
|
3193
1297
|
Add filtering conditions to the question.
|
|
3194
1298
|
Transform it to require WHERE clauses with specific conditions.
|
|
3195
1299
|
Examples:
|
|
@@ -3197,7 +1301,7 @@ var techniqueInstructions = {
|
|
|
3197
1301
|
- "List customers" → "List customers who have made more than 5 purchases"
|
|
3198
1302
|
- "Get products" → "Get products with price above $100"
|
|
3199
1303
|
`,
|
|
3200
|
-
"add-join":
|
|
1304
|
+
"add-join": dedent6`
|
|
3201
1305
|
Add requirements that need data from related tables.
|
|
3202
1306
|
Transform it to require JOIN operations between multiple tables.
|
|
3203
1307
|
Examples:
|
|
@@ -3205,7 +1309,7 @@ var techniqueInstructions = {
|
|
|
3205
1309
|
- "List products" → "List products with their supplier information"
|
|
3206
1310
|
- "Get employees" → "Get employees with their department and manager names"
|
|
3207
1311
|
`,
|
|
3208
|
-
"add-reasoning":
|
|
1312
|
+
"add-reasoning": dedent6`
|
|
3209
1313
|
Add multi-step reasoning requirements.
|
|
3210
1314
|
Transform it to require logical deduction, comparisons, or derived calculations.
|
|
3211
1315
|
Examples:
|
|
@@ -3213,7 +1317,7 @@ var techniqueInstructions = {
|
|
|
3213
1317
|
- "List products" → "Which products are underperforming compared to their category average?"
|
|
3214
1318
|
- "Get revenue" → "Which month had the highest growth compared to the previous month?"
|
|
3215
1319
|
`,
|
|
3216
|
-
hypothetical:
|
|
1320
|
+
hypothetical: dedent6`
|
|
3217
1321
|
Add a hypothetical or speculative scenario.
|
|
3218
1322
|
Transform it to require applying calculations or projections.
|
|
3219
1323
|
Examples:
|
|
@@ -3226,32 +1330,32 @@ var evolverOutputSchema = z7.object({
|
|
|
3226
1330
|
evolvedQuestion: z7.string().describe("The evolved, more complex version of the original question")
|
|
3227
1331
|
});
|
|
3228
1332
|
async function evolveQuestion(params) {
|
|
3229
|
-
const context = new
|
|
3230
|
-
store: new
|
|
1333
|
+
const context = new ContextEngine7({
|
|
1334
|
+
store: new InMemoryContextStore7(),
|
|
3231
1335
|
chatId: `evolver-${crypto.randomUUID()}`,
|
|
3232
1336
|
userId: "system"
|
|
3233
1337
|
});
|
|
3234
1338
|
context.set(
|
|
3235
|
-
|
|
1339
|
+
persona6({
|
|
3236
1340
|
name: "question_evolver",
|
|
3237
1341
|
role: "You are an expert at evolving simple database questions into more complex ones. Your task is to take a basic question and transform it into a more sophisticated version that requires advanced SQL techniques to answer.",
|
|
3238
1342
|
objective: "Transform simple questions into complex versions requiring advanced SQL techniques"
|
|
3239
1343
|
}),
|
|
3240
|
-
|
|
3241
|
-
|
|
1344
|
+
fragment6("original_question", params.question),
|
|
1345
|
+
fragment6(
|
|
3242
1346
|
"original_sql",
|
|
3243
1347
|
params.sql,
|
|
3244
1348
|
"(This shows what the original question required)"
|
|
3245
1349
|
),
|
|
3246
|
-
|
|
3247
|
-
|
|
1350
|
+
fragment6("database_schema", params.schema),
|
|
1351
|
+
fragment6(
|
|
3248
1352
|
"technique",
|
|
3249
1353
|
{ name: params.technique },
|
|
3250
1354
|
params.techniqueInstruction
|
|
3251
1355
|
),
|
|
3252
|
-
|
|
1356
|
+
fragment6(
|
|
3253
1357
|
"task",
|
|
3254
|
-
|
|
1358
|
+
dedent6`
|
|
3255
1359
|
Evolve the original question using the "${params.technique}" technique.
|
|
3256
1360
|
|
|
3257
1361
|
Requirements:
|
|
@@ -3263,22 +1367,22 @@ async function evolveQuestion(params) {
|
|
|
3263
1367
|
6. The evolved question should build upon the original topic/domain
|
|
3264
1368
|
`
|
|
3265
1369
|
),
|
|
3266
|
-
|
|
1370
|
+
guardrail3({
|
|
3267
1371
|
rule: "The evolved question MUST require more complex SQL than the original"
|
|
3268
1372
|
}),
|
|
3269
|
-
|
|
1373
|
+
guardrail3({
|
|
3270
1374
|
rule: "Do not ask for data that does not exist in the schema"
|
|
3271
1375
|
}),
|
|
3272
|
-
|
|
1376
|
+
guardrail3({
|
|
3273
1377
|
rule: "Keep the question grounded in the same domain as the original"
|
|
3274
1378
|
}),
|
|
3275
|
-
|
|
3276
|
-
|
|
1379
|
+
guardrail3({ rule: "Make sure the question is clear and unambiguous" }),
|
|
1380
|
+
user7(
|
|
3277
1381
|
`Evolve this question using "${params.technique}": "${params.question}"`
|
|
3278
1382
|
)
|
|
3279
1383
|
);
|
|
3280
|
-
const evolverOutput =
|
|
3281
|
-
model: params.model ??
|
|
1384
|
+
const evolverOutput = structuredOutput7({
|
|
1385
|
+
model: params.model ?? groq7("openai/gpt-oss-20b"),
|
|
3282
1386
|
context,
|
|
3283
1387
|
schema: evolverOutputSchema
|
|
3284
1388
|
});
|
|
@@ -3395,10 +1499,20 @@ async function withRetry2(computation) {
|
|
|
3395
1499
|
}
|
|
3396
1500
|
|
|
3397
1501
|
// packages/text2sql/src/lib/synthesis/synthesizers/persona-generator.ts
|
|
3398
|
-
import { groq as
|
|
3399
|
-
import
|
|
1502
|
+
import { groq as groq8 } from "@ai-sdk/groq";
|
|
1503
|
+
import dedent7 from "dedent";
|
|
3400
1504
|
import z8 from "zod";
|
|
3401
1505
|
import "@deepagents/agent";
|
|
1506
|
+
import {
|
|
1507
|
+
ContextEngine as ContextEngine8,
|
|
1508
|
+
InMemoryContextStore as InMemoryContextStore8,
|
|
1509
|
+
XmlRenderer,
|
|
1510
|
+
fragment as fragment7,
|
|
1511
|
+
guardrail as guardrail4,
|
|
1512
|
+
persona as personaFragment2,
|
|
1513
|
+
structuredOutput as structuredOutput8,
|
|
1514
|
+
user as user8
|
|
1515
|
+
} from "@deepagents/context";
|
|
3402
1516
|
var outputSchema3 = z8.object({
|
|
3403
1517
|
personas: z8.array(
|
|
3404
1518
|
z8.object({
|
|
@@ -3415,21 +1529,21 @@ var outputSchema3 = z8.object({
|
|
|
3415
1529
|
async function generatePersonas(schemaFragments, options) {
|
|
3416
1530
|
const schema = new XmlRenderer().render(schemaFragments);
|
|
3417
1531
|
const count = options?.count ?? 5;
|
|
3418
|
-
const context = new
|
|
3419
|
-
store: new
|
|
1532
|
+
const context = new ContextEngine8({
|
|
1533
|
+
store: new InMemoryContextStore8(),
|
|
3420
1534
|
chatId: `persona-gen-${crypto.randomUUID()}`,
|
|
3421
1535
|
userId: "system"
|
|
3422
1536
|
});
|
|
3423
1537
|
context.set(
|
|
3424
|
-
|
|
1538
|
+
personaFragment2({
|
|
3425
1539
|
name: "persona_generator",
|
|
3426
1540
|
role: "You are an expert at understanding database schemas and inferring who would use them.",
|
|
3427
1541
|
objective: "Generate realistic personas representing users who would query this database"
|
|
3428
1542
|
}),
|
|
3429
|
-
|
|
3430
|
-
|
|
1543
|
+
fragment7("database_schema", schema),
|
|
1544
|
+
fragment7(
|
|
3431
1545
|
"task",
|
|
3432
|
-
|
|
1546
|
+
dedent7`
|
|
3433
1547
|
Analyze the database schema and generate realistic personas representing
|
|
3434
1548
|
the different types of users who would query this database.
|
|
3435
1549
|
|
|
@@ -3460,9 +1574,9 @@ async function generatePersonas(schemaFragments, options) {
|
|
|
3460
1574
|
- Styles should match how this persona would naturally communicate
|
|
3461
1575
|
`
|
|
3462
1576
|
),
|
|
3463
|
-
|
|
1577
|
+
fragment7(
|
|
3464
1578
|
"example",
|
|
3465
|
-
|
|
1579
|
+
dedent7`
|
|
3466
1580
|
For an e-commerce schema with orders, customers, products tables:
|
|
3467
1581
|
|
|
3468
1582
|
{
|
|
@@ -3478,21 +1592,21 @@ async function generatePersonas(schemaFragments, options) {
|
|
|
3478
1592
|
}
|
|
3479
1593
|
`
|
|
3480
1594
|
),
|
|
3481
|
-
|
|
1595
|
+
guardrail4({
|
|
3482
1596
|
rule: "Only generate personas relevant to the actual schema provided"
|
|
3483
1597
|
}),
|
|
3484
|
-
|
|
1598
|
+
guardrail4({
|
|
3485
1599
|
rule: "Do not invent tables or data that do not exist in the schema"
|
|
3486
1600
|
}),
|
|
3487
|
-
|
|
1601
|
+
guardrail4({
|
|
3488
1602
|
rule: "Ensure perspectives are specific to the domain, not generic"
|
|
3489
1603
|
}),
|
|
3490
|
-
|
|
1604
|
+
user8(
|
|
3491
1605
|
`Generate exactly ${count} distinct personas who would query this database.`
|
|
3492
1606
|
)
|
|
3493
1607
|
);
|
|
3494
|
-
const personaOutput =
|
|
3495
|
-
model: options?.model ??
|
|
1608
|
+
const personaOutput = structuredOutput8({
|
|
1609
|
+
model: options?.model ?? groq8("openai/gpt-oss-20b"),
|
|
3496
1610
|
context,
|
|
3497
1611
|
schema: outputSchema3
|
|
3498
1612
|
});
|
|
@@ -3500,11 +1614,32 @@ async function generatePersonas(schemaFragments, options) {
|
|
|
3500
1614
|
return output.personas;
|
|
3501
1615
|
}
|
|
3502
1616
|
|
|
1617
|
+
// packages/text2sql/src/lib/synthesis/synthesizers/teachings-generator.ts
|
|
1618
|
+
import { XmlRenderer as XmlRenderer2 } from "@deepagents/context";
|
|
1619
|
+
|
|
3503
1620
|
// packages/text2sql/src/lib/agents/teachables.agent.ts
|
|
3504
|
-
import { groq as
|
|
3505
|
-
import
|
|
1621
|
+
import { groq as groq9 } from "@ai-sdk/groq";
|
|
1622
|
+
import dedent8 from "dedent";
|
|
3506
1623
|
import z9 from "zod";
|
|
3507
1624
|
import "@deepagents/agent";
|
|
1625
|
+
import {
|
|
1626
|
+
ContextEngine as ContextEngine9,
|
|
1627
|
+
InMemoryContextStore as InMemoryContextStore9,
|
|
1628
|
+
analogy,
|
|
1629
|
+
clarification,
|
|
1630
|
+
example,
|
|
1631
|
+
explain,
|
|
1632
|
+
fragment as fragment8,
|
|
1633
|
+
guardrail as guardrail5,
|
|
1634
|
+
hint,
|
|
1635
|
+
persona as persona7,
|
|
1636
|
+
quirk,
|
|
1637
|
+
structuredOutput as structuredOutput9,
|
|
1638
|
+
styleGuide,
|
|
1639
|
+
term,
|
|
1640
|
+
user as user9,
|
|
1641
|
+
workflow
|
|
1642
|
+
} from "@deepagents/context";
|
|
3508
1643
|
var outputSchema4 = z9.object({
|
|
3509
1644
|
terms: z9.array(z9.object({ name: z9.string(), definition: z9.string() })).optional().describe("Domain terminology definitions"),
|
|
3510
1645
|
hints: z9.array(z9.object({ text: z9.string() })).optional().describe("Helpful hints for SQL generation"),
|
|
@@ -3557,22 +1692,22 @@ var outputSchema4 = z9.object({
|
|
|
3557
1692
|
).optional().describe("Concept analogies")
|
|
3558
1693
|
});
|
|
3559
1694
|
async function toTeachings(input, options) {
|
|
3560
|
-
const context = new
|
|
3561
|
-
store: new
|
|
1695
|
+
const context = new ContextEngine9({
|
|
1696
|
+
store: new InMemoryContextStore9(),
|
|
3562
1697
|
chatId: `teachables-gen-${crypto.randomUUID()}`,
|
|
3563
1698
|
userId: "system"
|
|
3564
1699
|
});
|
|
3565
1700
|
context.set(
|
|
3566
|
-
|
|
1701
|
+
persona7({
|
|
3567
1702
|
name: "teachables-author",
|
|
3568
1703
|
role: 'You design "fragments" for a Text2SQL system. Fragments become structured XML instructions.',
|
|
3569
1704
|
objective: "Choose only high-impact items that improve accuracy, safety, or clarity for this database"
|
|
3570
1705
|
}),
|
|
3571
|
-
|
|
3572
|
-
...input.context ? [
|
|
3573
|
-
|
|
1706
|
+
fragment8("database_schema", input.schema),
|
|
1707
|
+
...input.context ? [fragment8("additional_context", input.context)] : [],
|
|
1708
|
+
fragment8(
|
|
3574
1709
|
"output_structure",
|
|
3575
|
-
|
|
1710
|
+
dedent8`
|
|
3576
1711
|
Output a JSON object with these optional arrays (include only relevant ones):
|
|
3577
1712
|
- terms: [{ name: string, definition: string }] - Domain terminology
|
|
3578
1713
|
- hints: [{ text: string }] - Helpful SQL generation hints
|
|
@@ -3586,9 +1721,9 @@ async function toTeachings(input, options) {
|
|
|
3586
1721
|
- analogies: [{ concepts: string[], relationship: string, insight?: string, therefore?: string, pitfall?: string }]
|
|
3587
1722
|
`
|
|
3588
1723
|
),
|
|
3589
|
-
|
|
1724
|
+
fragment8(
|
|
3590
1725
|
"task",
|
|
3591
|
-
|
|
1726
|
+
dedent8`
|
|
3592
1727
|
1. Analyze the schema to infer domain, relationships, and sensitive columns.
|
|
3593
1728
|
2. Generate 3-10 fragments total across all categories, prioritizing:
|
|
3594
1729
|
- guardrails for PII columns (email, ssn, phone, etc)
|
|
@@ -3598,12 +1733,12 @@ async function toTeachings(input, options) {
|
|
|
3598
1733
|
4. Only include categories that are relevant to this schema.
|
|
3599
1734
|
`
|
|
3600
1735
|
),
|
|
3601
|
-
|
|
1736
|
+
user9(
|
|
3602
1737
|
`Analyze this database schema and generate fragments that will help an AI generate accurate SQL queries.`
|
|
3603
1738
|
)
|
|
3604
1739
|
);
|
|
3605
|
-
const teachablesOutput =
|
|
3606
|
-
model: options?.model ??
|
|
1740
|
+
const teachablesOutput = structuredOutput9({
|
|
1741
|
+
model: options?.model ?? groq9("openai/gpt-oss-20b"),
|
|
3607
1742
|
context,
|
|
3608
1743
|
schema: outputSchema4
|
|
3609
1744
|
});
|
|
@@ -3613,7 +1748,7 @@ async function toTeachings(input, options) {
|
|
|
3613
1748
|
result.hints?.forEach((h) => fragments.push(hint(h.text)));
|
|
3614
1749
|
result.guardrails?.forEach(
|
|
3615
1750
|
(g) => fragments.push(
|
|
3616
|
-
|
|
1751
|
+
guardrail5({ rule: g.rule, reason: g.reason, action: g.action })
|
|
3617
1752
|
)
|
|
3618
1753
|
);
|
|
3619
1754
|
result.explains?.forEach(
|
|
@@ -3669,7 +1804,7 @@ async function toTeachings(input, options) {
|
|
|
3669
1804
|
|
|
3670
1805
|
// packages/text2sql/src/lib/synthesis/synthesizers/teachings-generator.ts
|
|
3671
1806
|
async function generateTeachings(schemaFragments, options) {
|
|
3672
|
-
const schema = new
|
|
1807
|
+
const schema = new XmlRenderer2().render(schemaFragments);
|
|
3673
1808
|
const maxRetries = options?.maxRetries ?? 3;
|
|
3674
1809
|
let lastError;
|
|
3675
1810
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|