@deepagents/text2sql 0.10.2 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -41
- package/dist/index.d.ts +1 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2338 -2398
- package/dist/index.js.map +4 -4
- package/dist/lib/adapters/adapter.d.ts +13 -1
- package/dist/lib/adapters/adapter.d.ts.map +1 -1
- package/dist/lib/adapters/groundings/abstract.grounding.d.ts +19 -3
- package/dist/lib/adapters/groundings/abstract.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/groundings/column-stats.grounding.d.ts +1 -2
- package/dist/lib/adapters/groundings/column-stats.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/groundings/column-values.grounding.d.ts +1 -2
- package/dist/lib/adapters/groundings/column-values.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/groundings/constraint.grounding.d.ts +1 -1
- package/dist/lib/adapters/groundings/constraint.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/groundings/index.js +15 -222
- package/dist/lib/adapters/groundings/index.js.map +3 -3
- package/dist/lib/adapters/groundings/indexes.grounding.d.ts +1 -1
- package/dist/lib/adapters/groundings/indexes.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/groundings/info.grounding.d.ts +1 -1
- package/dist/lib/adapters/groundings/info.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/groundings/report.grounding.d.ts +1 -1
- package/dist/lib/adapters/groundings/report.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/groundings/row-count.grounding.d.ts +1 -1
- package/dist/lib/adapters/groundings/row-count.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/groundings/table.grounding.d.ts +3 -3
- package/dist/lib/adapters/groundings/table.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/groundings/view.grounding.d.ts +1 -1
- package/dist/lib/adapters/groundings/view.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/mysql/index.js +343 -315
- package/dist/lib/adapters/mysql/index.js.map +4 -4
- package/dist/lib/adapters/postgres/index.js +385 -357
- package/dist/lib/adapters/postgres/index.js.map +4 -4
- package/dist/lib/adapters/spreadsheet/index.js +290 -223
- package/dist/lib/adapters/spreadsheet/index.js.map +4 -4
- package/dist/lib/adapters/sqlite/index.js +307 -279
- package/dist/lib/adapters/sqlite/index.js.map +4 -4
- package/dist/lib/adapters/sqlserver/index.js +383 -355
- package/dist/lib/adapters/sqlserver/index.js.map +4 -4
- package/dist/lib/agents/developer.agent.d.ts +33 -23
- package/dist/lib/agents/developer.agent.d.ts.map +1 -1
- package/dist/lib/agents/sql.agent.d.ts +4 -4
- package/dist/lib/agents/sql.agent.d.ts.map +1 -1
- package/dist/lib/agents/teachables.agent.d.ts +2 -2
- package/dist/lib/agents/teachables.agent.d.ts.map +1 -1
- package/dist/lib/agents/text2sql.agent.d.ts +18 -71
- package/dist/lib/agents/text2sql.agent.d.ts.map +1 -1
- package/dist/lib/fragments/schema.d.ts +214 -0
- package/dist/lib/fragments/schema.d.ts.map +1 -0
- package/dist/lib/instructions.d.ts +29 -2
- package/dist/lib/instructions.d.ts.map +1 -1
- package/dist/lib/instructions.js +336 -319
- package/dist/lib/instructions.js.map +4 -4
- package/dist/lib/sql.d.ts +13 -103
- package/dist/lib/sql.d.ts.map +1 -1
- package/dist/lib/synthesis/extractors/base-contextual-extractor.d.ts +2 -2
- package/dist/lib/synthesis/extractors/base-contextual-extractor.d.ts.map +1 -1
- package/dist/lib/synthesis/extractors/message-extractor.d.ts +1 -2
- package/dist/lib/synthesis/extractors/message-extractor.d.ts.map +1 -1
- package/dist/lib/synthesis/extractors/sql-extractor.d.ts.map +1 -1
- package/dist/lib/synthesis/index.js +1794 -572
- package/dist/lib/synthesis/index.js.map +4 -4
- package/dist/lib/synthesis/synthesizers/depth-evolver.d.ts.map +1 -1
- package/dist/lib/synthesis/synthesizers/persona-generator.d.ts +7 -17
- package/dist/lib/synthesis/synthesizers/persona-generator.d.ts.map +1 -1
- package/dist/lib/synthesis/synthesizers/schema-synthesizer.d.ts +2 -2
- package/dist/lib/synthesis/synthesizers/schema-synthesizer.d.ts.map +1 -1
- package/dist/lib/synthesis/synthesizers/teachings-generator.d.ts +8 -20
- package/dist/lib/synthesis/synthesizers/teachings-generator.d.ts.map +1 -1
- package/dist/lib/teach/teachings.d.ts +2 -2
- package/dist/lib/teach/teachings.d.ts.map +1 -1
- package/package.json +4 -3
- package/dist/lib/agents/chat1.agent.d.ts +0 -50
- package/dist/lib/agents/chat1.agent.d.ts.map +0 -1
- package/dist/lib/agents/chat2.agent.d.ts +0 -68
- package/dist/lib/agents/chat2.agent.d.ts.map +0 -1
- package/dist/lib/agents/chat3.agent.d.ts +0 -80
- package/dist/lib/agents/chat3.agent.d.ts.map +0 -1
- package/dist/lib/agents/chat4.agent.d.ts +0 -88
- package/dist/lib/agents/chat4.agent.d.ts.map +0 -1
- package/dist/lib/history/history.d.ts +0 -41
- package/dist/lib/history/history.d.ts.map +0 -1
- package/dist/lib/history/memory.history.d.ts +0 -5
- package/dist/lib/history/memory.history.d.ts.map +0 -1
- package/dist/lib/history/sqlite.history.d.ts +0 -15
- package/dist/lib/history/sqlite.history.d.ts.map +0 -1
- package/dist/lib/memory/memory.prompt.d.ts +0 -3
- package/dist/lib/memory/memory.prompt.d.ts.map +0 -1
- package/dist/lib/memory/memory.store.d.ts +0 -5
- package/dist/lib/memory/memory.store.d.ts.map +0 -1
- package/dist/lib/memory/sqlite.store.d.ts +0 -14
- package/dist/lib/memory/sqlite.store.d.ts.map +0 -1
- package/dist/lib/memory/store.d.ts +0 -40
- package/dist/lib/memory/store.d.ts.map +0 -1
- package/dist/lib/teach/teachables.d.ts +0 -648
- package/dist/lib/teach/teachables.d.ts.map +0 -1
- package/dist/lib/teach/xml.d.ts +0 -6
- package/dist/lib/teach/xml.d.ts.map +0 -1
|
@@ -220,22 +220,25 @@ ${state.introspection}
|
|
|
220
220
|
</examples>
|
|
221
221
|
`
|
|
222
222
|
});
|
|
223
|
-
function getMessageText(
|
|
224
|
-
const textParts =
|
|
223
|
+
function getMessageText(message2) {
|
|
224
|
+
const textParts = message2.parts.filter(isTextUIPart).map((part) => part.text);
|
|
225
225
|
return textParts.join(" ").trim();
|
|
226
226
|
}
|
|
227
227
|
function formatConversation(messages) {
|
|
228
228
|
return messages.map((msg, i) => `[${i + 1}] ${msg}`).join("\n");
|
|
229
229
|
}
|
|
230
230
|
var BaseContextualExtractor = class extends PairProducer {
|
|
231
|
+
context = [];
|
|
232
|
+
results = [];
|
|
233
|
+
messages;
|
|
234
|
+
adapter;
|
|
235
|
+
options;
|
|
231
236
|
constructor(messages, adapter, options = {}) {
|
|
232
237
|
super();
|
|
233
238
|
this.messages = messages;
|
|
234
239
|
this.adapter = adapter;
|
|
235
240
|
this.options = options;
|
|
236
241
|
}
|
|
237
|
-
context = [];
|
|
238
|
-
results = [];
|
|
239
242
|
/**
|
|
240
243
|
* Template method - defines the extraction algorithm skeleton.
|
|
241
244
|
* Subclasses customize behavior via hooks, not by overriding this method.
|
|
@@ -248,31 +251,31 @@ var BaseContextualExtractor = class extends PairProducer {
|
|
|
248
251
|
if (this.results.length === 0) {
|
|
249
252
|
return;
|
|
250
253
|
}
|
|
251
|
-
const introspection =
|
|
254
|
+
const introspection = "";
|
|
252
255
|
yield* this.resolveQuestions(introspection);
|
|
253
256
|
}
|
|
254
257
|
/**
|
|
255
258
|
* Core extraction loop - iterates through messages and calls hooks.
|
|
256
259
|
*/
|
|
257
260
|
async extractSqlsWithContext(toolName, includeFailures) {
|
|
258
|
-
for (const
|
|
259
|
-
if (
|
|
260
|
-
const text = getMessageText(
|
|
261
|
+
for (const message2 of this.messages) {
|
|
262
|
+
if (message2.role === "user") {
|
|
263
|
+
const text = getMessageText(message2);
|
|
261
264
|
if (text) {
|
|
262
265
|
await this.onUserMessage(text);
|
|
263
266
|
}
|
|
264
267
|
continue;
|
|
265
268
|
}
|
|
266
|
-
if (
|
|
267
|
-
await this.extractFromAssistant(
|
|
269
|
+
if (message2.role === "assistant") {
|
|
270
|
+
await this.extractFromAssistant(message2, toolName, includeFailures);
|
|
268
271
|
}
|
|
269
272
|
}
|
|
270
273
|
}
|
|
271
274
|
/**
|
|
272
275
|
* Extract SQL from assistant message parts.
|
|
273
276
|
*/
|
|
274
|
-
async extractFromAssistant(
|
|
275
|
-
for (const part of
|
|
277
|
+
async extractFromAssistant(message2, toolName, includeFailures) {
|
|
278
|
+
for (const part of message2.parts) {
|
|
276
279
|
if (!isToolOrDynamicToolUIPart(part)) {
|
|
277
280
|
continue;
|
|
278
281
|
}
|
|
@@ -301,7 +304,7 @@ var BaseContextualExtractor = class extends PairProducer {
|
|
|
301
304
|
conversationContext: snapshot
|
|
302
305
|
});
|
|
303
306
|
}
|
|
304
|
-
const assistantText = getMessageText(
|
|
307
|
+
const assistantText = getMessageText(message2);
|
|
305
308
|
if (assistantText) {
|
|
306
309
|
this.context.push(`Assistant: ${assistantText}`);
|
|
307
310
|
}
|
|
@@ -334,29 +337,31 @@ var BaseContextualExtractor = class extends PairProducer {
|
|
|
334
337
|
|
|
335
338
|
// packages/text2sql/src/lib/synthesis/extractors/message-extractor.ts
|
|
336
339
|
var MessageExtractor = class extends PairProducer {
|
|
340
|
+
#messages;
|
|
341
|
+
#options;
|
|
337
342
|
/**
|
|
338
343
|
* @param messages - Chat history to extract pairs from
|
|
339
344
|
* @param options - Extraction configuration
|
|
340
345
|
*/
|
|
341
346
|
constructor(messages, options = {}) {
|
|
342
347
|
super();
|
|
343
|
-
this
|
|
344
|
-
this
|
|
348
|
+
this.#messages = messages;
|
|
349
|
+
this.#options = options;
|
|
345
350
|
}
|
|
346
351
|
/**
|
|
347
352
|
* Extracts question-SQL pairs by parsing tool calls and pairing with user messages.
|
|
348
353
|
* @returns Pairs extracted from db_query tool invocations
|
|
349
354
|
*/
|
|
350
355
|
async *produce() {
|
|
351
|
-
const { includeFailures = false, toolName = "db_query" } = this
|
|
356
|
+
const { includeFailures = false, toolName = "db_query" } = this.#options;
|
|
352
357
|
let lastUserMessage = null;
|
|
353
|
-
for (const
|
|
354
|
-
if (
|
|
355
|
-
lastUserMessage =
|
|
358
|
+
for (const message2 of this.#messages) {
|
|
359
|
+
if (message2.role === "user") {
|
|
360
|
+
lastUserMessage = message2;
|
|
356
361
|
continue;
|
|
357
362
|
}
|
|
358
|
-
if (
|
|
359
|
-
for (const part of
|
|
363
|
+
if (message2.role === "assistant" && lastUserMessage) {
|
|
364
|
+
for (const part of message2.parts) {
|
|
360
365
|
if (!isToolOrDynamicToolUIPart2(part)) {
|
|
361
366
|
continue;
|
|
362
367
|
}
|
|
@@ -459,7 +464,7 @@ var SqlExtractor = class extends PairProducer {
|
|
|
459
464
|
*/
|
|
460
465
|
async *produce() {
|
|
461
466
|
const { validateSql = true, skipInvalid = false } = this.#options;
|
|
462
|
-
const introspection =
|
|
467
|
+
const introspection = "";
|
|
463
468
|
for (const sql of this.#sqls) {
|
|
464
469
|
let isValid = true;
|
|
465
470
|
if (validateSql) {
|
|
@@ -822,377 +827,1595 @@ import { Console } from "node:console";
|
|
|
822
827
|
import { createWriteStream } from "node:fs";
|
|
823
828
|
import pRetry from "p-retry";
|
|
824
829
|
import z5 from "zod";
|
|
830
|
+
import "@deepagents/agent";
|
|
831
|
+
|
|
832
|
+
// packages/context/dist/index.js
|
|
833
|
+
import { encode } from "gpt-tokenizer";
|
|
834
|
+
import { generateId } from "ai";
|
|
835
|
+
import pluralize from "pluralize";
|
|
836
|
+
import { titlecase } from "stringcase";
|
|
837
|
+
import { defineCommand } from "just-bash";
|
|
838
|
+
import spawn from "nano-spawn";
|
|
839
|
+
import "bash-tool";
|
|
840
|
+
import spawn2 from "nano-spawn";
|
|
825
841
|
import {
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
842
|
+
createBashTool
|
|
843
|
+
} from "bash-tool";
|
|
844
|
+
import YAML from "yaml";
|
|
845
|
+
import { DatabaseSync } from "node:sqlite";
|
|
846
|
+
import {
|
|
847
|
+
Output,
|
|
848
|
+
convertToModelMessages,
|
|
849
|
+
createUIMessageStream,
|
|
850
|
+
generateId as generateId2,
|
|
851
|
+
generateText,
|
|
852
|
+
smoothStream,
|
|
853
|
+
stepCountIs,
|
|
854
|
+
streamText
|
|
855
|
+
} from "ai";
|
|
856
|
+
import chalk from "chalk";
|
|
857
|
+
import "zod";
|
|
858
|
+
import "@deepagents/agent";
|
|
859
|
+
var defaultTokenizer = {
|
|
860
|
+
encode(text) {
|
|
861
|
+
return encode(text);
|
|
862
|
+
},
|
|
863
|
+
count(text) {
|
|
864
|
+
return encode(text).length;
|
|
837
865
|
}
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
866
|
+
};
|
|
867
|
+
var ModelsRegistry = class {
|
|
868
|
+
#cache = /* @__PURE__ */ new Map();
|
|
869
|
+
#loaded = false;
|
|
870
|
+
#tokenizers = /* @__PURE__ */ new Map();
|
|
871
|
+
#defaultTokenizer = defaultTokenizer;
|
|
872
|
+
/**
|
|
873
|
+
* Load models data from models.dev API
|
|
874
|
+
*/
|
|
875
|
+
async load() {
|
|
876
|
+
if (this.#loaded) return;
|
|
877
|
+
const response = await fetch("https://models.dev/api.json");
|
|
878
|
+
if (!response.ok) {
|
|
879
|
+
throw new Error(`Failed to fetch models: ${response.statusText}`);
|
|
880
|
+
}
|
|
881
|
+
const data = await response.json();
|
|
882
|
+
for (const [providerId, provider] of Object.entries(data)) {
|
|
883
|
+
for (const [modelId, model] of Object.entries(provider.models)) {
|
|
884
|
+
const info = {
|
|
885
|
+
id: model.id,
|
|
886
|
+
name: model.name,
|
|
887
|
+
family: model.family,
|
|
888
|
+
cost: model.cost,
|
|
889
|
+
limit: model.limit,
|
|
890
|
+
provider: providerId
|
|
891
|
+
};
|
|
892
|
+
this.#cache.set(`${providerId}:${modelId}`, info);
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
this.#loaded = true;
|
|
896
|
+
}
|
|
897
|
+
/**
|
|
898
|
+
* Get model info by ID
|
|
899
|
+
* @param modelId - Model ID (e.g., "openai:gpt-4o")
|
|
900
|
+
*/
|
|
901
|
+
get(modelId) {
|
|
902
|
+
return this.#cache.get(modelId);
|
|
903
|
+
}
|
|
904
|
+
/**
|
|
905
|
+
* Check if a model exists in the registry
|
|
906
|
+
*/
|
|
907
|
+
has(modelId) {
|
|
908
|
+
return this.#cache.has(modelId);
|
|
909
|
+
}
|
|
910
|
+
/**
|
|
911
|
+
* List all available model IDs
|
|
912
|
+
*/
|
|
913
|
+
list() {
|
|
914
|
+
return [...this.#cache.keys()];
|
|
915
|
+
}
|
|
916
|
+
/**
|
|
917
|
+
* Register a custom tokenizer for specific model families
|
|
918
|
+
* @param family - Model family name (e.g., "llama", "claude")
|
|
919
|
+
* @param tokenizer - Tokenizer implementation
|
|
920
|
+
*/
|
|
921
|
+
registerTokenizer(family, tokenizer) {
|
|
922
|
+
this.#tokenizers.set(family, tokenizer);
|
|
923
|
+
}
|
|
924
|
+
/**
|
|
925
|
+
* Set the default tokenizer used when no family-specific tokenizer is registered
|
|
926
|
+
*/
|
|
927
|
+
setDefaultTokenizer(tokenizer) {
|
|
928
|
+
this.#defaultTokenizer = tokenizer;
|
|
929
|
+
}
|
|
930
|
+
/**
|
|
931
|
+
* Get the appropriate tokenizer for a model
|
|
932
|
+
*/
|
|
933
|
+
getTokenizer(modelId) {
|
|
934
|
+
const model = this.get(modelId);
|
|
935
|
+
if (model) {
|
|
936
|
+
const familyTokenizer = this.#tokenizers.get(model.family);
|
|
937
|
+
if (familyTokenizer) {
|
|
938
|
+
return familyTokenizer;
|
|
939
|
+
}
|
|
940
|
+
}
|
|
941
|
+
return this.#defaultTokenizer;
|
|
942
|
+
}
|
|
943
|
+
/**
|
|
944
|
+
* Estimate token count and cost for given text and model
|
|
945
|
+
* @param modelId - Model ID to use for pricing (e.g., "openai:gpt-4o")
|
|
946
|
+
* @param input - Input text (prompt)
|
|
947
|
+
*/
|
|
948
|
+
estimate(modelId, input) {
|
|
949
|
+
const model = this.get(modelId);
|
|
950
|
+
if (!model) {
|
|
951
|
+
throw new Error(
|
|
952
|
+
`Model "${modelId}" not found. Call load() first or check model ID.`
|
|
953
|
+
);
|
|
954
|
+
}
|
|
955
|
+
const tokenizer = this.getTokenizer(modelId);
|
|
956
|
+
const tokens = tokenizer.count(input);
|
|
957
|
+
const cost = tokens / 1e6 * model.cost.input;
|
|
958
|
+
return {
|
|
959
|
+
model: model.id,
|
|
960
|
+
provider: model.provider,
|
|
961
|
+
tokens,
|
|
962
|
+
cost,
|
|
963
|
+
limits: {
|
|
964
|
+
context: model.limit.context,
|
|
965
|
+
output: model.limit.output,
|
|
966
|
+
exceedsContext: tokens > model.limit.context
|
|
967
|
+
},
|
|
968
|
+
fragments: []
|
|
969
|
+
};
|
|
970
|
+
}
|
|
971
|
+
};
|
|
972
|
+
var _registry = null;
|
|
973
|
+
function getModelsRegistry() {
|
|
974
|
+
if (!_registry) {
|
|
975
|
+
_registry = new ModelsRegistry();
|
|
976
|
+
}
|
|
977
|
+
return _registry;
|
|
978
|
+
}
|
|
979
|
+
function isFragment(data) {
|
|
980
|
+
return typeof data === "object" && data !== null && "name" in data && "data" in data && typeof data.name === "string";
|
|
981
|
+
}
|
|
982
|
+
function isFragmentObject(data) {
|
|
983
|
+
return typeof data === "object" && data !== null && !Array.isArray(data) && !isFragment(data);
|
|
984
|
+
}
|
|
985
|
+
function isMessageFragment(fragment2) {
|
|
986
|
+
return fragment2.type === "message";
|
|
987
|
+
}
|
|
988
|
+
function role(content) {
|
|
989
|
+
return {
|
|
990
|
+
name: "role",
|
|
991
|
+
data: content
|
|
992
|
+
};
|
|
993
|
+
}
|
|
994
|
+
function user6(content) {
|
|
995
|
+
const message2 = typeof content === "string" ? {
|
|
996
|
+
id: generateId(),
|
|
997
|
+
role: "user",
|
|
998
|
+
parts: [{ type: "text", text: content }]
|
|
999
|
+
} : content;
|
|
1000
|
+
return {
|
|
1001
|
+
id: message2.id,
|
|
1002
|
+
name: "user",
|
|
1003
|
+
data: "content",
|
|
1004
|
+
type: "message",
|
|
1005
|
+
persist: true,
|
|
1006
|
+
codec: {
|
|
1007
|
+
decode() {
|
|
1008
|
+
return message2;
|
|
1009
|
+
},
|
|
1010
|
+
encode() {
|
|
1011
|
+
return message2;
|
|
1012
|
+
}
|
|
1013
|
+
}
|
|
1014
|
+
};
|
|
841
1015
|
}
|
|
842
|
-
function
|
|
843
|
-
|
|
844
|
-
|
|
1016
|
+
function message(content) {
|
|
1017
|
+
const message2 = typeof content === "string" ? {
|
|
1018
|
+
id: generateId(),
|
|
1019
|
+
role: "user",
|
|
1020
|
+
parts: [{ type: "text", text: content }]
|
|
1021
|
+
} : content;
|
|
1022
|
+
return {
|
|
1023
|
+
id: message2.id,
|
|
1024
|
+
name: "message",
|
|
1025
|
+
data: "content",
|
|
1026
|
+
type: "message",
|
|
1027
|
+
persist: true,
|
|
1028
|
+
codec: {
|
|
1029
|
+
decode() {
|
|
1030
|
+
return message2;
|
|
1031
|
+
},
|
|
1032
|
+
encode() {
|
|
1033
|
+
return message2;
|
|
1034
|
+
}
|
|
1035
|
+
}
|
|
1036
|
+
};
|
|
1037
|
+
}
|
|
1038
|
+
var ContextRenderer = class {
|
|
1039
|
+
options;
|
|
1040
|
+
constructor(options = {}) {
|
|
1041
|
+
this.options = options;
|
|
1042
|
+
}
|
|
1043
|
+
/**
|
|
1044
|
+
* Check if data is a primitive (string, number, boolean).
|
|
1045
|
+
*/
|
|
1046
|
+
isPrimitive(data) {
|
|
1047
|
+
return typeof data === "string" || typeof data === "number" || typeof data === "boolean";
|
|
1048
|
+
}
|
|
1049
|
+
/**
|
|
1050
|
+
* Group fragments by name for groupFragments option.
|
|
1051
|
+
*/
|
|
1052
|
+
groupByName(fragments) {
|
|
1053
|
+
const groups = /* @__PURE__ */ new Map();
|
|
1054
|
+
for (const fragment2 of fragments) {
|
|
1055
|
+
const existing = groups.get(fragment2.name) ?? [];
|
|
1056
|
+
existing.push(fragment2);
|
|
1057
|
+
groups.set(fragment2.name, existing);
|
|
1058
|
+
}
|
|
1059
|
+
return groups;
|
|
1060
|
+
}
|
|
1061
|
+
/**
|
|
1062
|
+
* Template method - dispatches value to appropriate handler.
|
|
1063
|
+
*/
|
|
1064
|
+
renderValue(key, value, ctx) {
|
|
1065
|
+
if (value == null) {
|
|
1066
|
+
return "";
|
|
1067
|
+
}
|
|
1068
|
+
if (isFragment(value)) {
|
|
1069
|
+
return this.renderFragment(value, ctx);
|
|
1070
|
+
}
|
|
1071
|
+
if (Array.isArray(value)) {
|
|
1072
|
+
return this.renderArray(key, value, ctx);
|
|
1073
|
+
}
|
|
1074
|
+
if (isFragmentObject(value)) {
|
|
1075
|
+
return this.renderObject(key, value, ctx);
|
|
1076
|
+
}
|
|
1077
|
+
return this.renderPrimitive(key, String(value), ctx);
|
|
845
1078
|
}
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
1079
|
+
/**
|
|
1080
|
+
* Render all entries of an object.
|
|
1081
|
+
*/
|
|
1082
|
+
renderEntries(data, ctx) {
|
|
1083
|
+
return Object.entries(data).map(([key, value]) => this.renderValue(key, value, ctx)).filter(Boolean);
|
|
1084
|
+
}
|
|
1085
|
+
};
|
|
1086
|
+
var XmlRenderer = class extends ContextRenderer {
|
|
1087
|
+
render(fragments) {
|
|
1088
|
+
return fragments.map((f) => this.#renderTopLevel(f)).filter(Boolean).join("\n");
|
|
1089
|
+
}
|
|
1090
|
+
#renderTopLevel(fragment2) {
|
|
1091
|
+
if (this.isPrimitive(fragment2.data)) {
|
|
1092
|
+
return this.#leafRoot(fragment2.name, String(fragment2.data));
|
|
1093
|
+
}
|
|
1094
|
+
if (Array.isArray(fragment2.data)) {
|
|
1095
|
+
return this.#renderArray(fragment2.name, fragment2.data, 0);
|
|
1096
|
+
}
|
|
1097
|
+
if (isFragment(fragment2.data)) {
|
|
1098
|
+
const child = this.renderFragment(fragment2.data, { depth: 1, path: [] });
|
|
1099
|
+
return this.#wrap(fragment2.name, [child]);
|
|
1100
|
+
}
|
|
1101
|
+
return this.#wrap(
|
|
1102
|
+
fragment2.name,
|
|
1103
|
+
this.renderEntries(fragment2.data, { depth: 1, path: [] })
|
|
1104
|
+
);
|
|
1105
|
+
}
|
|
1106
|
+
#renderArray(name, items, depth) {
|
|
1107
|
+
const fragmentItems = items.filter(isFragment);
|
|
1108
|
+
const nonFragmentItems = items.filter((item) => !isFragment(item));
|
|
1109
|
+
const children = [];
|
|
1110
|
+
for (const item of nonFragmentItems) {
|
|
1111
|
+
if (item != null) {
|
|
1112
|
+
children.push(
|
|
1113
|
+
this.#leaf(pluralize.singular(name), String(item), depth + 1)
|
|
1114
|
+
);
|
|
1115
|
+
}
|
|
1116
|
+
}
|
|
1117
|
+
if (this.options.groupFragments && fragmentItems.length > 0) {
|
|
1118
|
+
const groups = this.groupByName(fragmentItems);
|
|
1119
|
+
for (const [groupName, groupFragments] of groups) {
|
|
1120
|
+
const groupChildren = groupFragments.map(
|
|
1121
|
+
(frag) => this.renderFragment(frag, { depth: depth + 2, path: [] })
|
|
1122
|
+
);
|
|
1123
|
+
const pluralName = pluralize.plural(groupName);
|
|
1124
|
+
children.push(this.#wrapIndented(pluralName, groupChildren, depth + 1));
|
|
1125
|
+
}
|
|
1126
|
+
} else {
|
|
1127
|
+
for (const frag of fragmentItems) {
|
|
1128
|
+
children.push(
|
|
1129
|
+
this.renderFragment(frag, { depth: depth + 1, path: [] })
|
|
1130
|
+
);
|
|
1131
|
+
}
|
|
1132
|
+
}
|
|
1133
|
+
return this.#wrap(name, children);
|
|
1134
|
+
}
|
|
1135
|
+
#leafRoot(tag, value) {
|
|
1136
|
+
const safe = this.#escape(value);
|
|
1137
|
+
if (safe.includes("\n")) {
|
|
1138
|
+
return `<${tag}>
|
|
1139
|
+
${this.#indent(safe, 2)}
|
|
849
1140
|
</${tag}>`;
|
|
850
|
-
}
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
1141
|
+
}
|
|
1142
|
+
return `<${tag}>${safe}</${tag}>`;
|
|
1143
|
+
}
|
|
1144
|
+
renderFragment(fragment2, ctx) {
|
|
1145
|
+
const { name, data } = fragment2;
|
|
1146
|
+
if (this.isPrimitive(data)) {
|
|
1147
|
+
return this.#leaf(name, String(data), ctx.depth);
|
|
1148
|
+
}
|
|
1149
|
+
if (isFragment(data)) {
|
|
1150
|
+
const child = this.renderFragment(data, { ...ctx, depth: ctx.depth + 1 });
|
|
1151
|
+
return this.#wrapIndented(name, [child], ctx.depth);
|
|
1152
|
+
}
|
|
1153
|
+
if (Array.isArray(data)) {
|
|
1154
|
+
return this.#renderArrayIndented(name, data, ctx.depth);
|
|
1155
|
+
}
|
|
1156
|
+
const children = this.renderEntries(data, { ...ctx, depth: ctx.depth + 1 });
|
|
1157
|
+
return this.#wrapIndented(name, children, ctx.depth);
|
|
1158
|
+
}
|
|
1159
|
+
#renderArrayIndented(name, items, depth) {
|
|
1160
|
+
const fragmentItems = items.filter(isFragment);
|
|
1161
|
+
const nonFragmentItems = items.filter((item) => !isFragment(item));
|
|
1162
|
+
const children = [];
|
|
1163
|
+
for (const item of nonFragmentItems) {
|
|
1164
|
+
if (item != null) {
|
|
1165
|
+
children.push(
|
|
1166
|
+
this.#leaf(pluralize.singular(name), String(item), depth + 1)
|
|
1167
|
+
);
|
|
1168
|
+
}
|
|
1169
|
+
}
|
|
1170
|
+
if (this.options.groupFragments && fragmentItems.length > 0) {
|
|
1171
|
+
const groups = this.groupByName(fragmentItems);
|
|
1172
|
+
for (const [groupName, groupFragments] of groups) {
|
|
1173
|
+
const groupChildren = groupFragments.map(
|
|
1174
|
+
(frag) => this.renderFragment(frag, { depth: depth + 2, path: [] })
|
|
1175
|
+
);
|
|
1176
|
+
const pluralName = pluralize.plural(groupName);
|
|
1177
|
+
children.push(this.#wrapIndented(pluralName, groupChildren, depth + 1));
|
|
1178
|
+
}
|
|
1179
|
+
} else {
|
|
1180
|
+
for (const frag of fragmentItems) {
|
|
1181
|
+
children.push(
|
|
1182
|
+
this.renderFragment(frag, { depth: depth + 1, path: [] })
|
|
1183
|
+
);
|
|
1184
|
+
}
|
|
1185
|
+
}
|
|
1186
|
+
return this.#wrapIndented(name, children, depth);
|
|
1187
|
+
}
|
|
1188
|
+
renderPrimitive(key, value, ctx) {
|
|
1189
|
+
return this.#leaf(key, value, ctx.depth);
|
|
1190
|
+
}
|
|
1191
|
+
renderArray(key, items, ctx) {
|
|
1192
|
+
if (!items.length) {
|
|
1193
|
+
return "";
|
|
1194
|
+
}
|
|
1195
|
+
const itemTag = pluralize.singular(key);
|
|
1196
|
+
const children = items.filter((item) => item != null).map((item) => this.#leaf(itemTag, String(item), ctx.depth + 1));
|
|
1197
|
+
return this.#wrapIndented(key, children, ctx.depth);
|
|
1198
|
+
}
|
|
1199
|
+
renderObject(key, obj, ctx) {
|
|
1200
|
+
const children = this.renderEntries(obj, { ...ctx, depth: ctx.depth + 1 });
|
|
1201
|
+
return this.#wrapIndented(key, children, ctx.depth);
|
|
1202
|
+
}
|
|
1203
|
+
#escape(value) {
|
|
1204
|
+
if (value == null) {
|
|
1205
|
+
return "";
|
|
1206
|
+
}
|
|
1207
|
+
return value.replaceAll(/&/g, "&").replaceAll(/</g, "<").replaceAll(/>/g, ">").replaceAll(/"/g, """).replaceAll(/'/g, "'");
|
|
1208
|
+
}
|
|
1209
|
+
#indent(text, spaces) {
|
|
1210
|
+
if (!text.trim()) {
|
|
1211
|
+
return "";
|
|
1212
|
+
}
|
|
1213
|
+
const padding = " ".repeat(spaces);
|
|
1214
|
+
return text.split("\n").map((line) => line.length ? padding + line : padding).join("\n");
|
|
1215
|
+
}
|
|
1216
|
+
#leaf(tag, value, depth) {
|
|
1217
|
+
const safe = this.#escape(value);
|
|
1218
|
+
const pad = " ".repeat(depth);
|
|
1219
|
+
if (safe.includes("\n")) {
|
|
1220
|
+
return `${pad}<${tag}>
|
|
1221
|
+
${this.#indent(safe, (depth + 1) * 2)}
|
|
1222
|
+
${pad}</${tag}>`;
|
|
1223
|
+
}
|
|
1224
|
+
return `${pad}<${tag}>${safe}</${tag}>`;
|
|
1225
|
+
}
|
|
1226
|
+
#wrap(tag, children) {
|
|
1227
|
+
const content = children.filter(Boolean).join("\n");
|
|
1228
|
+
if (!content) {
|
|
1229
|
+
return "";
|
|
1230
|
+
}
|
|
854
1231
|
return `<${tag}>
|
|
855
|
-
${
|
|
1232
|
+
${content}
|
|
856
1233
|
</${tag}>`;
|
|
857
1234
|
}
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
1235
|
+
#wrapIndented(tag, children, depth) {
|
|
1236
|
+
const content = children.filter(Boolean).join("\n");
|
|
1237
|
+
if (!content) {
|
|
1238
|
+
return "";
|
|
1239
|
+
}
|
|
1240
|
+
const pad = " ".repeat(depth);
|
|
1241
|
+
return `${pad}<${tag}>
|
|
1242
|
+
${content}
|
|
1243
|
+
${pad}</${tag}>`;
|
|
863
1244
|
}
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
}
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
1245
|
+
};
|
|
1246
|
+
var ContextStore = class {
|
|
1247
|
+
};
|
|
1248
|
+
var ContextEngine = class {
|
|
1249
|
+
/** Non-message fragments (role, hints, etc.) - not persisted in graph */
|
|
1250
|
+
#fragments = [];
|
|
1251
|
+
/** Pending message fragments to be added to graph */
|
|
1252
|
+
#pendingMessages = [];
|
|
1253
|
+
#store;
|
|
1254
|
+
#chatId;
|
|
1255
|
+
#branchName;
|
|
1256
|
+
#branch = null;
|
|
1257
|
+
#chatData = null;
|
|
1258
|
+
#initialized = false;
|
|
1259
|
+
constructor(options) {
|
|
1260
|
+
if (!options.chatId) {
|
|
1261
|
+
throw new Error("chatId is required");
|
|
1262
|
+
}
|
|
1263
|
+
this.#store = options.store;
|
|
1264
|
+
this.#chatId = options.chatId;
|
|
1265
|
+
this.#branchName = options.branch ?? "main";
|
|
870
1266
|
}
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
1267
|
+
/**
|
|
1268
|
+
* Initialize the chat and branch if they don't exist.
|
|
1269
|
+
*/
|
|
1270
|
+
async #ensureInitialized() {
|
|
1271
|
+
if (this.#initialized) {
|
|
1272
|
+
return;
|
|
1273
|
+
}
|
|
1274
|
+
this.#chatData = await this.#store.upsertChat({ id: this.#chatId });
|
|
1275
|
+
const existingBranch = await this.#store.getBranch(
|
|
1276
|
+
this.#chatId,
|
|
1277
|
+
this.#branchName
|
|
1278
|
+
);
|
|
1279
|
+
if (existingBranch) {
|
|
1280
|
+
this.#branch = existingBranch;
|
|
1281
|
+
} else {
|
|
1282
|
+
this.#branch = {
|
|
1283
|
+
id: crypto.randomUUID(),
|
|
1284
|
+
chatId: this.#chatId,
|
|
1285
|
+
name: this.#branchName,
|
|
1286
|
+
headMessageId: null,
|
|
1287
|
+
isActive: true,
|
|
1288
|
+
createdAt: Date.now()
|
|
1289
|
+
};
|
|
1290
|
+
await this.#store.createBranch(this.#branch);
|
|
1291
|
+
}
|
|
1292
|
+
this.#initialized = true;
|
|
1293
|
+
}
|
|
1294
|
+
/**
|
|
1295
|
+
* Create a new branch from a specific message.
|
|
1296
|
+
* Shared logic between rewind() and btw().
|
|
1297
|
+
*/
|
|
1298
|
+
async #createBranchFrom(messageId, switchTo) {
|
|
1299
|
+
const branches = await this.#store.listBranches(this.#chatId);
|
|
1300
|
+
const samePrefix = branches.filter(
|
|
1301
|
+
(b) => b.name === this.#branchName || b.name.startsWith(`${this.#branchName}-v`)
|
|
1302
|
+
);
|
|
1303
|
+
const newBranchName = `${this.#branchName}-v${samePrefix.length + 1}`;
|
|
1304
|
+
const newBranch = {
|
|
1305
|
+
id: crypto.randomUUID(),
|
|
1306
|
+
chatId: this.#chatId,
|
|
1307
|
+
name: newBranchName,
|
|
1308
|
+
headMessageId: messageId,
|
|
1309
|
+
isActive: false,
|
|
1310
|
+
createdAt: Date.now()
|
|
1311
|
+
};
|
|
1312
|
+
await this.#store.createBranch(newBranch);
|
|
1313
|
+
if (switchTo) {
|
|
1314
|
+
await this.#store.setActiveBranch(this.#chatId, newBranch.id);
|
|
1315
|
+
this.#branch = { ...newBranch, isActive: true };
|
|
1316
|
+
this.#branchName = newBranchName;
|
|
1317
|
+
this.#pendingMessages = [];
|
|
1318
|
+
}
|
|
1319
|
+
const chain = await this.#store.getMessageChain(messageId);
|
|
1320
|
+
return {
|
|
1321
|
+
id: newBranch.id,
|
|
1322
|
+
name: newBranch.name,
|
|
1323
|
+
headMessageId: newBranch.headMessageId,
|
|
1324
|
+
isActive: switchTo,
|
|
1325
|
+
messageCount: chain.length,
|
|
1326
|
+
createdAt: newBranch.createdAt
|
|
1327
|
+
};
|
|
1328
|
+
}
|
|
1329
|
+
/**
|
|
1330
|
+
* Get the current chat ID.
|
|
1331
|
+
*/
|
|
1332
|
+
get chatId() {
|
|
1333
|
+
return this.#chatId;
|
|
1334
|
+
}
|
|
1335
|
+
/**
|
|
1336
|
+
* Get the current branch name.
|
|
1337
|
+
*/
|
|
1338
|
+
get branch() {
|
|
1339
|
+
return this.#branchName;
|
|
1340
|
+
}
|
|
1341
|
+
/**
|
|
1342
|
+
* Get metadata for the current chat.
|
|
1343
|
+
* Returns null if the chat hasn't been initialized yet.
|
|
1344
|
+
*/
|
|
1345
|
+
get chat() {
|
|
1346
|
+
if (!this.#chatData) {
|
|
1347
|
+
return null;
|
|
1348
|
+
}
|
|
1349
|
+
return {
|
|
1350
|
+
id: this.#chatData.id,
|
|
1351
|
+
createdAt: this.#chatData.createdAt,
|
|
1352
|
+
updatedAt: this.#chatData.updatedAt,
|
|
1353
|
+
title: this.#chatData.title,
|
|
1354
|
+
metadata: this.#chatData.metadata
|
|
1355
|
+
};
|
|
1356
|
+
}
|
|
1357
|
+
/**
|
|
1358
|
+
* Add fragments to the context.
|
|
1359
|
+
*
|
|
1360
|
+
* - Message fragments (user/assistant) are queued for persistence
|
|
1361
|
+
* - Non-message fragments (role/hint) are kept in memory for system prompt
|
|
1362
|
+
*/
|
|
1363
|
+
set(...fragments) {
|
|
1364
|
+
for (const fragment2 of fragments) {
|
|
1365
|
+
if (isMessageFragment(fragment2)) {
|
|
1366
|
+
this.#pendingMessages.push(fragment2);
|
|
1367
|
+
} else {
|
|
1368
|
+
this.#fragments.push(fragment2);
|
|
1369
|
+
}
|
|
1370
|
+
}
|
|
1371
|
+
return this;
|
|
1372
|
+
}
|
|
1373
|
+
// Unset a fragment by ID (not implemented yet)
|
|
1374
|
+
unset(fragmentId) {
|
|
1375
|
+
}
|
|
1376
|
+
/**
|
|
1377
|
+
* Render all fragments using the provided renderer.
|
|
1378
|
+
* @internal Use resolve() instead for public API.
|
|
1379
|
+
*/
|
|
1380
|
+
render(renderer) {
|
|
1381
|
+
return renderer.render(this.#fragments);
|
|
1382
|
+
}
|
|
1383
|
+
/**
|
|
1384
|
+
* Resolve context into AI SDK-ready format.
|
|
1385
|
+
*
|
|
1386
|
+
* - Initializes chat and branch if needed
|
|
1387
|
+
* - Loads message history from the graph (walking parent chain)
|
|
1388
|
+
* - Separates context fragments for system prompt
|
|
1389
|
+
* - Combines with pending messages
|
|
1390
|
+
*
|
|
1391
|
+
* @example
|
|
1392
|
+
* ```ts
|
|
1393
|
+
* const context = new ContextEngine({ store, chatId: 'chat-1' })
|
|
1394
|
+
* .set(role('You are helpful'), user('Hello'));
|
|
1395
|
+
*
|
|
1396
|
+
* const { systemPrompt, messages } = await context.resolve();
|
|
1397
|
+
* await generateText({ system: systemPrompt, messages });
|
|
1398
|
+
* ```
|
|
1399
|
+
*/
|
|
1400
|
+
async resolve(options) {
|
|
1401
|
+
await this.#ensureInitialized();
|
|
1402
|
+
const systemPrompt = options.renderer.render(this.#fragments);
|
|
1403
|
+
const messages = [];
|
|
1404
|
+
if (this.#branch?.headMessageId) {
|
|
1405
|
+
const chain = await this.#store.getMessageChain(
|
|
1406
|
+
this.#branch.headMessageId
|
|
1407
|
+
);
|
|
1408
|
+
for (const msg of chain) {
|
|
1409
|
+
messages.push(message(msg.data).codec?.decode());
|
|
1410
|
+
}
|
|
1411
|
+
}
|
|
1412
|
+
for (const fragment2 of this.#pendingMessages) {
|
|
1413
|
+
const decoded = fragment2.codec.decode();
|
|
1414
|
+
messages.push(decoded);
|
|
1415
|
+
}
|
|
1416
|
+
return { systemPrompt, messages };
|
|
1417
|
+
}
|
|
1418
|
+
/**
|
|
1419
|
+
* Save pending messages to the graph.
|
|
1420
|
+
*
|
|
1421
|
+
* Each message is added as a node with parentId pointing to the previous message.
|
|
1422
|
+
* The branch head is updated to point to the last message.
|
|
1423
|
+
*
|
|
1424
|
+
* @example
|
|
1425
|
+
* ```ts
|
|
1426
|
+
* context.set(user('Hello'));
|
|
1427
|
+
* // AI responds...
|
|
1428
|
+
* context.set(assistant('Hi there!'));
|
|
1429
|
+
* await context.save(); // Persist to graph
|
|
1430
|
+
* ```
|
|
1431
|
+
*/
|
|
1432
|
+
async save() {
|
|
1433
|
+
await this.#ensureInitialized();
|
|
1434
|
+
if (this.#pendingMessages.length === 0) {
|
|
1435
|
+
return;
|
|
1436
|
+
}
|
|
1437
|
+
let parentId = this.#branch.headMessageId;
|
|
1438
|
+
const now = Date.now();
|
|
1439
|
+
for (const fragment2 of this.#pendingMessages) {
|
|
1440
|
+
const messageData = {
|
|
1441
|
+
id: fragment2.id ?? crypto.randomUUID(),
|
|
1442
|
+
chatId: this.#chatId,
|
|
1443
|
+
parentId,
|
|
1444
|
+
name: fragment2.name,
|
|
1445
|
+
type: fragment2.type,
|
|
1446
|
+
data: fragment2.codec.encode(),
|
|
1447
|
+
createdAt: now
|
|
1448
|
+
};
|
|
1449
|
+
await this.#store.addMessage(messageData);
|
|
1450
|
+
parentId = messageData.id;
|
|
1451
|
+
}
|
|
1452
|
+
await this.#store.updateBranchHead(this.#branch.id, parentId);
|
|
1453
|
+
this.#branch.headMessageId = parentId;
|
|
1454
|
+
this.#pendingMessages = [];
|
|
1455
|
+
}
|
|
1456
|
+
/**
|
|
1457
|
+
* Estimate token count and cost for the full context.
|
|
1458
|
+
*
|
|
1459
|
+
* Includes:
|
|
1460
|
+
* - System prompt fragments (role, hints, etc.)
|
|
1461
|
+
* - Persisted chat messages (from store)
|
|
1462
|
+
* - Pending messages (not yet saved)
|
|
1463
|
+
*
|
|
1464
|
+
* @param modelId - Model ID (e.g., "openai:gpt-4o", "anthropic:claude-3-5-sonnet")
|
|
1465
|
+
* @param options - Optional settings
|
|
1466
|
+
* @returns Estimate result with token counts, costs, and per-fragment breakdown
|
|
1467
|
+
*/
|
|
1468
|
+
async estimate(modelId, options = {}) {
|
|
1469
|
+
await this.#ensureInitialized();
|
|
1470
|
+
const renderer = options.renderer ?? new XmlRenderer();
|
|
1471
|
+
const registry = getModelsRegistry();
|
|
1472
|
+
await registry.load();
|
|
1473
|
+
const model = registry.get(modelId);
|
|
1474
|
+
if (!model) {
|
|
1475
|
+
throw new Error(
|
|
1476
|
+
`Model "${modelId}" not found. Call load() first or check model ID.`
|
|
1477
|
+
);
|
|
1478
|
+
}
|
|
1479
|
+
const tokenizer = registry.getTokenizer(modelId);
|
|
1480
|
+
const fragmentEstimates = [];
|
|
1481
|
+
for (const fragment2 of this.#fragments) {
|
|
1482
|
+
const rendered = renderer.render([fragment2]);
|
|
1483
|
+
const tokens = tokenizer.count(rendered);
|
|
1484
|
+
const cost = tokens / 1e6 * model.cost.input;
|
|
1485
|
+
fragmentEstimates.push({
|
|
1486
|
+
id: fragment2.id,
|
|
1487
|
+
name: fragment2.name,
|
|
1488
|
+
tokens,
|
|
1489
|
+
cost
|
|
1490
|
+
});
|
|
1491
|
+
}
|
|
1492
|
+
if (this.#branch?.headMessageId) {
|
|
1493
|
+
const chain = await this.#store.getMessageChain(
|
|
1494
|
+
this.#branch.headMessageId
|
|
1495
|
+
);
|
|
1496
|
+
for (const msg of chain) {
|
|
1497
|
+
const content = String(msg.data);
|
|
1498
|
+
const tokens = tokenizer.count(content);
|
|
1499
|
+
const cost = tokens / 1e6 * model.cost.input;
|
|
1500
|
+
fragmentEstimates.push({
|
|
1501
|
+
name: msg.name,
|
|
1502
|
+
id: msg.id,
|
|
1503
|
+
tokens,
|
|
1504
|
+
cost
|
|
1505
|
+
});
|
|
1506
|
+
}
|
|
1507
|
+
}
|
|
1508
|
+
for (const fragment2 of this.#pendingMessages) {
|
|
1509
|
+
const content = String(fragment2.data);
|
|
1510
|
+
const tokens = tokenizer.count(content);
|
|
1511
|
+
const cost = tokens / 1e6 * model.cost.input;
|
|
1512
|
+
fragmentEstimates.push({
|
|
1513
|
+
name: fragment2.name,
|
|
1514
|
+
id: fragment2.id,
|
|
1515
|
+
tokens,
|
|
1516
|
+
cost
|
|
1517
|
+
});
|
|
1518
|
+
}
|
|
1519
|
+
const totalTokens = fragmentEstimates.reduce((sum, f) => sum + f.tokens, 0);
|
|
1520
|
+
const totalCost = fragmentEstimates.reduce((sum, f) => sum + f.cost, 0);
|
|
1521
|
+
return {
|
|
1522
|
+
model: model.id,
|
|
1523
|
+
provider: model.provider,
|
|
1524
|
+
tokens: totalTokens,
|
|
1525
|
+
cost: totalCost,
|
|
1526
|
+
limits: {
|
|
1527
|
+
context: model.limit.context,
|
|
1528
|
+
output: model.limit.output,
|
|
1529
|
+
exceedsContext: totalTokens > model.limit.context
|
|
1530
|
+
},
|
|
1531
|
+
fragments: fragmentEstimates
|
|
1532
|
+
};
|
|
1533
|
+
}
|
|
1534
|
+
/**
|
|
1535
|
+
* Rewind to a specific message by ID.
|
|
1536
|
+
*
|
|
1537
|
+
* Creates a new branch from that message, preserving the original branch.
|
|
1538
|
+
* The new branch becomes active.
|
|
1539
|
+
*
|
|
1540
|
+
* @param messageId - The message ID to rewind to
|
|
1541
|
+
* @returns The new branch info
|
|
1542
|
+
*
|
|
1543
|
+
* @example
|
|
1544
|
+
* ```ts
|
|
1545
|
+
* context.set(user('What is 2 + 2?', { id: 'q1' }));
|
|
1546
|
+
* context.set(assistant('The answer is 5.', { id: 'wrong' })); // Oops!
|
|
1547
|
+
* await context.save();
|
|
1548
|
+
*
|
|
1549
|
+
* // Rewind to the question, creates new branch
|
|
1550
|
+
* const newBranch = await context.rewind('q1');
|
|
1551
|
+
*
|
|
1552
|
+
* // Now add correct answer on new branch
|
|
1553
|
+
* context.set(assistant('The answer is 4.'));
|
|
1554
|
+
* await context.save();
|
|
1555
|
+
* ```
|
|
1556
|
+
*/
|
|
1557
|
+
async rewind(messageId) {
|
|
1558
|
+
await this.#ensureInitialized();
|
|
1559
|
+
const message2 = await this.#store.getMessage(messageId);
|
|
1560
|
+
if (!message2) {
|
|
1561
|
+
throw new Error(`Message "${messageId}" not found`);
|
|
1562
|
+
}
|
|
1563
|
+
if (message2.chatId !== this.#chatId) {
|
|
1564
|
+
throw new Error(`Message "${messageId}" belongs to a different chat`);
|
|
1565
|
+
}
|
|
1566
|
+
return this.#createBranchFrom(messageId, true);
|
|
1567
|
+
}
|
|
1568
|
+
/**
|
|
1569
|
+
* Create a checkpoint at the current position.
|
|
1570
|
+
*
|
|
1571
|
+
* A checkpoint is a named pointer to the current branch head.
|
|
1572
|
+
* Use restore() to return to this point later.
|
|
1573
|
+
*
|
|
1574
|
+
* @param name - Name for the checkpoint
|
|
1575
|
+
* @returns The checkpoint info
|
|
1576
|
+
*
|
|
1577
|
+
* @example
|
|
1578
|
+
* ```ts
|
|
1579
|
+
* context.set(user('I want to learn a new skill.'));
|
|
1580
|
+
* context.set(assistant('Would you like coding or cooking?'));
|
|
1581
|
+
* await context.save();
|
|
1582
|
+
*
|
|
1583
|
+
* // Save checkpoint before user's choice
|
|
1584
|
+
* const cp = await context.checkpoint('before-choice');
|
|
1585
|
+
* ```
|
|
1586
|
+
*/
|
|
1587
|
+
async checkpoint(name) {
|
|
1588
|
+
await this.#ensureInitialized();
|
|
1589
|
+
if (!this.#branch?.headMessageId) {
|
|
1590
|
+
throw new Error("Cannot create checkpoint: no messages in conversation");
|
|
1591
|
+
}
|
|
1592
|
+
const checkpoint = {
|
|
1593
|
+
id: crypto.randomUUID(),
|
|
1594
|
+
chatId: this.#chatId,
|
|
1595
|
+
name,
|
|
1596
|
+
messageId: this.#branch.headMessageId,
|
|
1597
|
+
createdAt: Date.now()
|
|
1598
|
+
};
|
|
1599
|
+
await this.#store.createCheckpoint(checkpoint);
|
|
1600
|
+
return {
|
|
1601
|
+
id: checkpoint.id,
|
|
1602
|
+
name: checkpoint.name,
|
|
1603
|
+
messageId: checkpoint.messageId,
|
|
1604
|
+
createdAt: checkpoint.createdAt
|
|
1605
|
+
};
|
|
1606
|
+
}
|
|
1607
|
+
/**
|
|
1608
|
+
* Restore to a checkpoint by creating a new branch from that point.
|
|
1609
|
+
*
|
|
1610
|
+
* @param name - Name of the checkpoint to restore
|
|
1611
|
+
* @returns The new branch info
|
|
1612
|
+
*
|
|
1613
|
+
* @example
|
|
1614
|
+
* ```ts
|
|
1615
|
+
* // User chose cooking, but wants to try coding path
|
|
1616
|
+
* await context.restore('before-choice');
|
|
1617
|
+
*
|
|
1618
|
+
* context.set(user('I want to learn coding.'));
|
|
1619
|
+
* context.set(assistant('Python is a great starting language!'));
|
|
1620
|
+
* await context.save();
|
|
1621
|
+
* ```
|
|
1622
|
+
*/
|
|
1623
|
+
async restore(name) {
|
|
1624
|
+
await this.#ensureInitialized();
|
|
1625
|
+
const checkpoint = await this.#store.getCheckpoint(this.#chatId, name);
|
|
1626
|
+
if (!checkpoint) {
|
|
1627
|
+
throw new Error(
|
|
1628
|
+
`Checkpoint "${name}" not found in chat "${this.#chatId}"`
|
|
1629
|
+
);
|
|
1630
|
+
}
|
|
1631
|
+
return this.rewind(checkpoint.messageId);
|
|
1632
|
+
}
|
|
1633
|
+
/**
|
|
1634
|
+
* Switch to a different branch by name.
|
|
1635
|
+
*
|
|
1636
|
+
* @param name - Branch name to switch to
|
|
1637
|
+
*
|
|
1638
|
+
* @example
|
|
1639
|
+
* ```ts
|
|
1640
|
+
* // List branches (via store)
|
|
1641
|
+
* const branches = await store.listBranches(context.chatId);
|
|
1642
|
+
* console.log(branches); // [{name: 'main', ...}, {name: 'main-v2', ...}]
|
|
1643
|
+
*
|
|
1644
|
+
* // Switch to original branch
|
|
1645
|
+
* await context.switchBranch('main');
|
|
1646
|
+
* ```
|
|
1647
|
+
*/
|
|
1648
|
+
async switchBranch(name) {
|
|
1649
|
+
await this.#ensureInitialized();
|
|
1650
|
+
const branch = await this.#store.getBranch(this.#chatId, name);
|
|
1651
|
+
if (!branch) {
|
|
1652
|
+
throw new Error(`Branch "${name}" not found in chat "${this.#chatId}"`);
|
|
1653
|
+
}
|
|
1654
|
+
await this.#store.setActiveBranch(this.#chatId, branch.id);
|
|
1655
|
+
this.#branch = { ...branch, isActive: true };
|
|
1656
|
+
this.#branchName = name;
|
|
1657
|
+
this.#pendingMessages = [];
|
|
1658
|
+
}
|
|
1659
|
+
/**
|
|
1660
|
+
* Create a parallel branch from the current position ("by the way").
|
|
1661
|
+
*
|
|
1662
|
+
* Use this when you want to fork the conversation without leaving
|
|
1663
|
+
* the current branch. Common use case: user wants to ask another
|
|
1664
|
+
* question while waiting for the model to respond.
|
|
1665
|
+
*
|
|
1666
|
+
* Unlike rewind(), this method:
|
|
1667
|
+
* - Uses the current HEAD (no messageId needed)
|
|
1668
|
+
* - Does NOT switch to the new branch
|
|
1669
|
+
* - Keeps pending messages intact
|
|
1670
|
+
*
|
|
1671
|
+
* @returns The new branch info (does not switch to it)
|
|
1672
|
+
* @throws Error if no messages exist in the conversation
|
|
1673
|
+
*
|
|
1674
|
+
* @example
|
|
1675
|
+
* ```ts
|
|
1676
|
+
* // User asked a question, model is generating...
|
|
1677
|
+
* context.set(user('What is the weather?'));
|
|
1678
|
+
* await context.save();
|
|
1679
|
+
*
|
|
1680
|
+
* // User wants to ask something else without waiting
|
|
1681
|
+
* const newBranch = await context.btw();
|
|
1682
|
+
* // newBranch = { name: 'main-v2', ... }
|
|
1683
|
+
*
|
|
1684
|
+
* // Later, switch to the new branch and add the question
|
|
1685
|
+
* await context.switchBranch(newBranch.name);
|
|
1686
|
+
* context.set(user('Also, what time is it?'));
|
|
1687
|
+
* await context.save();
|
|
1688
|
+
* ```
|
|
1689
|
+
*/
|
|
1690
|
+
async btw() {
|
|
1691
|
+
await this.#ensureInitialized();
|
|
1692
|
+
if (!this.#branch?.headMessageId) {
|
|
1693
|
+
throw new Error("Cannot create btw branch: no messages in conversation");
|
|
1694
|
+
}
|
|
1695
|
+
return this.#createBranchFrom(this.#branch.headMessageId, false);
|
|
1696
|
+
}
|
|
1697
|
+
/**
|
|
1698
|
+
* Update metadata for the current chat.
|
|
1699
|
+
*
|
|
1700
|
+
* @param updates - Partial metadata to merge (title, metadata)
|
|
1701
|
+
*
|
|
1702
|
+
* @example
|
|
1703
|
+
* ```ts
|
|
1704
|
+
* await context.updateChat({
|
|
1705
|
+
* title: 'Coding Help Session',
|
|
1706
|
+
* metadata: { tags: ['python', 'debugging'] }
|
|
1707
|
+
* });
|
|
1708
|
+
* ```
|
|
1709
|
+
*/
|
|
1710
|
+
async updateChat(updates) {
|
|
1711
|
+
await this.#ensureInitialized();
|
|
1712
|
+
const storeUpdates = {};
|
|
1713
|
+
if (updates.title !== void 0) {
|
|
1714
|
+
storeUpdates.title = updates.title;
|
|
1715
|
+
}
|
|
1716
|
+
if (updates.metadata !== void 0) {
|
|
1717
|
+
storeUpdates.metadata = {
|
|
1718
|
+
...this.#chatData?.metadata,
|
|
1719
|
+
...updates.metadata
|
|
1720
|
+
};
|
|
1721
|
+
}
|
|
1722
|
+
this.#chatData = await this.#store.updateChat(this.#chatId, storeUpdates);
|
|
1723
|
+
}
|
|
1724
|
+
/**
|
|
1725
|
+
* Consolidate context fragments (no-op for now).
|
|
1726
|
+
*
|
|
1727
|
+
* This is a placeholder for future functionality that merges context fragments
|
|
1728
|
+
* using specific rules. Currently, it does nothing.
|
|
1729
|
+
*
|
|
1730
|
+
* @experimental
|
|
1731
|
+
*/
|
|
1732
|
+
consolidate() {
|
|
1733
|
+
return void 0;
|
|
1734
|
+
}
|
|
1735
|
+
/**
|
|
1736
|
+
* Inspect the full context state for debugging.
|
|
1737
|
+
* Returns a comprehensive JSON-serializable object with all context information.
|
|
1738
|
+
*
|
|
1739
|
+
* @param options - Inspection options (modelId and renderer required)
|
|
1740
|
+
* @returns Complete inspection data including estimates, rendered output, fragments, and graph
|
|
1741
|
+
*
|
|
1742
|
+
* @example
|
|
1743
|
+
* ```ts
|
|
1744
|
+
* const inspection = await context.inspect({
|
|
1745
|
+
* modelId: 'openai:gpt-4o',
|
|
1746
|
+
* renderer: new XmlRenderer(),
|
|
1747
|
+
* });
|
|
1748
|
+
* console.log(JSON.stringify(inspection, null, 2));
|
|
1749
|
+
*
|
|
1750
|
+
* // Or write to file for analysis
|
|
1751
|
+
* await fs.writeFile('context-debug.json', JSON.stringify(inspection, null, 2));
|
|
1752
|
+
* ```
|
|
1753
|
+
*/
|
|
1754
|
+
async inspect(options) {
|
|
1755
|
+
await this.#ensureInitialized();
|
|
1756
|
+
const { renderer } = options;
|
|
1757
|
+
const estimateResult = await this.estimate(options.modelId, { renderer });
|
|
1758
|
+
const rendered = renderer.render(this.#fragments);
|
|
1759
|
+
const persistedMessages = [];
|
|
1760
|
+
if (this.#branch?.headMessageId) {
|
|
1761
|
+
const chain = await this.#store.getMessageChain(
|
|
1762
|
+
this.#branch.headMessageId
|
|
1763
|
+
);
|
|
1764
|
+
persistedMessages.push(...chain);
|
|
1765
|
+
}
|
|
1766
|
+
const graph = await this.#store.getGraph(this.#chatId);
|
|
1767
|
+
return {
|
|
1768
|
+
estimate: estimateResult,
|
|
1769
|
+
rendered,
|
|
1770
|
+
fragments: {
|
|
1771
|
+
context: [...this.#fragments],
|
|
1772
|
+
pending: [...this.#pendingMessages],
|
|
1773
|
+
persisted: persistedMessages
|
|
1774
|
+
},
|
|
1775
|
+
graph,
|
|
1776
|
+
meta: {
|
|
1777
|
+
chatId: this.#chatId,
|
|
1778
|
+
branch: this.#branchName,
|
|
1779
|
+
timestamp: Date.now()
|
|
1780
|
+
}
|
|
1781
|
+
};
|
|
1782
|
+
}
|
|
1783
|
+
};
|
|
875
1784
|
function term(name, definition) {
|
|
876
1785
|
return {
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
decode: () => wrapBlock("term", [leaf("name", name), leaf("definition", definition)])
|
|
1786
|
+
name: "term",
|
|
1787
|
+
data: { name, definition }
|
|
880
1788
|
};
|
|
881
1789
|
}
|
|
882
1790
|
function hint(text) {
|
|
883
1791
|
return {
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
decode: () => leaf("hint", text)
|
|
1792
|
+
name: "hint",
|
|
1793
|
+
data: text
|
|
887
1794
|
};
|
|
888
1795
|
}
|
|
889
1796
|
function guardrail(input) {
|
|
890
|
-
const { rule, reason, action } = input;
|
|
891
1797
|
return {
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
])
|
|
1798
|
+
name: "guardrail",
|
|
1799
|
+
data: {
|
|
1800
|
+
rule: input.rule,
|
|
1801
|
+
...input.reason && { reason: input.reason },
|
|
1802
|
+
...input.action && { action: input.action }
|
|
1803
|
+
}
|
|
899
1804
|
};
|
|
900
1805
|
}
|
|
901
1806
|
function explain(input) {
|
|
902
|
-
const { concept, explanation, therefore } = input;
|
|
903
1807
|
return {
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
])
|
|
1808
|
+
name: "explain",
|
|
1809
|
+
data: {
|
|
1810
|
+
concept: input.concept,
|
|
1811
|
+
explanation: input.explanation,
|
|
1812
|
+
...input.therefore && { therefore: input.therefore }
|
|
1813
|
+
}
|
|
911
1814
|
};
|
|
912
1815
|
}
|
|
913
1816
|
function example(input) {
|
|
914
|
-
const { question, answer, note } = input;
|
|
915
1817
|
return {
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
])
|
|
1818
|
+
name: "example",
|
|
1819
|
+
data: {
|
|
1820
|
+
question: input.question,
|
|
1821
|
+
answer: input.answer,
|
|
1822
|
+
...input.note && { note: input.note }
|
|
1823
|
+
}
|
|
923
1824
|
};
|
|
924
1825
|
}
|
|
925
1826
|
function clarification(input) {
|
|
926
|
-
const { when, ask, reason } = input;
|
|
927
1827
|
return {
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
])
|
|
1828
|
+
name: "clarification",
|
|
1829
|
+
data: {
|
|
1830
|
+
when: input.when,
|
|
1831
|
+
ask: input.ask,
|
|
1832
|
+
reason: input.reason
|
|
1833
|
+
}
|
|
935
1834
|
};
|
|
936
1835
|
}
|
|
937
1836
|
function workflow(input) {
|
|
938
|
-
const { task, steps, triggers, notes } = input;
|
|
939
1837
|
return {
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
triggers?.length
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
])
|
|
1838
|
+
name: "workflow",
|
|
1839
|
+
data: {
|
|
1840
|
+
task: input.task,
|
|
1841
|
+
steps: input.steps,
|
|
1842
|
+
...input.triggers?.length && { triggers: input.triggers },
|
|
1843
|
+
...input.notes && { notes: input.notes }
|
|
1844
|
+
}
|
|
948
1845
|
};
|
|
949
1846
|
}
|
|
950
1847
|
function quirk(input) {
|
|
951
|
-
const { issue, workaround } = input;
|
|
952
1848
|
return {
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
])
|
|
1849
|
+
name: "quirk",
|
|
1850
|
+
data: {
|
|
1851
|
+
issue: input.issue,
|
|
1852
|
+
workaround: input.workaround
|
|
1853
|
+
}
|
|
959
1854
|
};
|
|
960
1855
|
}
|
|
961
1856
|
function styleGuide(input) {
|
|
962
|
-
const { prefer, never, always } = input;
|
|
963
1857
|
return {
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
always
|
|
969
|
-
|
|
970
|
-
])
|
|
1858
|
+
name: "styleGuide",
|
|
1859
|
+
data: {
|
|
1860
|
+
prefer: input.prefer,
|
|
1861
|
+
...input.never && { never: input.never },
|
|
1862
|
+
...input.always && { always: input.always }
|
|
1863
|
+
}
|
|
971
1864
|
};
|
|
972
1865
|
}
|
|
973
1866
|
function analogy(input) {
|
|
974
|
-
const { concept, relationship, insight, therefore, pitfall } = input;
|
|
975
|
-
return {
|
|
976
|
-
type: "analogy",
|
|
977
|
-
encode: () => ({
|
|
978
|
-
type: "analogy",
|
|
979
|
-
concept,
|
|
980
|
-
relationship,
|
|
981
|
-
insight,
|
|
982
|
-
therefore,
|
|
983
|
-
pitfall
|
|
984
|
-
}),
|
|
985
|
-
decode: () => wrapBlock("analogy", [
|
|
986
|
-
list("concepts", concept, "concept"),
|
|
987
|
-
leaf("relationship", relationship),
|
|
988
|
-
insight ? leaf("insight", insight) : "",
|
|
989
|
-
therefore ? leaf("therefore", therefore) : "",
|
|
990
|
-
pitfall ? leaf("pitfall", pitfall) : ""
|
|
991
|
-
])
|
|
992
|
-
};
|
|
993
|
-
}
|
|
994
|
-
function glossary(entries) {
|
|
995
|
-
return {
|
|
996
|
-
type: "glossary",
|
|
997
|
-
encode: () => ({ type: "glossary", entries }),
|
|
998
|
-
decode: () => wrapBlock(
|
|
999
|
-
"glossary",
|
|
1000
|
-
Object.entries(entries).map(
|
|
1001
|
-
([term2, sql]) => wrapBlock("entry", [leaf("term", term2), leaf("sql", sql)])
|
|
1002
|
-
)
|
|
1003
|
-
)
|
|
1004
|
-
};
|
|
1005
|
-
}
|
|
1006
|
-
function identity(input) {
|
|
1007
|
-
const { name, role } = input;
|
|
1008
1867
|
return {
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1868
|
+
name: "analogy",
|
|
1869
|
+
data: {
|
|
1870
|
+
concepts: input.concepts,
|
|
1871
|
+
relationship: input.relationship,
|
|
1872
|
+
...input.insight && { insight: input.insight },
|
|
1873
|
+
...input.therefore && { therefore: input.therefore },
|
|
1874
|
+
...input.pitfall && { pitfall: input.pitfall }
|
|
1875
|
+
}
|
|
1015
1876
|
};
|
|
1016
1877
|
}
|
|
1017
1878
|
function persona(input) {
|
|
1018
|
-
const { name, role, tone } = input;
|
|
1019
1879
|
return {
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
])
|
|
1027
|
-
};
|
|
1028
|
-
}
|
|
1029
|
-
function alias(termName, meaning) {
|
|
1030
|
-
return {
|
|
1031
|
-
type: "alias",
|
|
1032
|
-
encode: () => ({ type: "alias", term: termName, meaning }),
|
|
1033
|
-
decode: () => wrapBlock("alias", [leaf("term", termName), leaf("meaning", meaning)])
|
|
1034
|
-
};
|
|
1035
|
-
}
|
|
1036
|
-
function preference(aspect, value) {
|
|
1037
|
-
return {
|
|
1038
|
-
type: "preference",
|
|
1039
|
-
encode: () => ({ type: "preference", aspect, value }),
|
|
1040
|
-
decode: () => wrapBlock("preference", [leaf("aspect", aspect), leaf("value", value)])
|
|
1041
|
-
};
|
|
1042
|
-
}
|
|
1043
|
-
function context(description) {
|
|
1044
|
-
return {
|
|
1045
|
-
type: "context",
|
|
1046
|
-
encode: () => ({ type: "context", description }),
|
|
1047
|
-
decode: () => leaf("context", description)
|
|
1048
|
-
};
|
|
1049
|
-
}
|
|
1050
|
-
function correction(subject, clarification2) {
|
|
1051
|
-
return {
|
|
1052
|
-
type: "correction",
|
|
1053
|
-
encode: () => ({ type: "correction", subject, clarification: clarification2 }),
|
|
1054
|
-
decode: () => wrapBlock("correction", [
|
|
1055
|
-
leaf("subject", subject),
|
|
1056
|
-
leaf("clarification", clarification2)
|
|
1057
|
-
])
|
|
1880
|
+
name: "persona",
|
|
1881
|
+
data: {
|
|
1882
|
+
name: input.name,
|
|
1883
|
+
role: input.role,
|
|
1884
|
+
...input.tone && { tone: input.tone }
|
|
1885
|
+
}
|
|
1058
1886
|
};
|
|
1059
1887
|
}
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1888
|
+
var STORE_DDL = `
|
|
1889
|
+
-- Chats table
|
|
1890
|
+
-- createdAt/updatedAt: DEFAULT for insert, inline SET for updates
|
|
1891
|
+
CREATE TABLE IF NOT EXISTS chats (
|
|
1892
|
+
id TEXT PRIMARY KEY,
|
|
1893
|
+
title TEXT,
|
|
1894
|
+
metadata TEXT,
|
|
1895
|
+
createdAt INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
|
|
1896
|
+
updatedAt INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000)
|
|
1897
|
+
);
|
|
1898
|
+
|
|
1899
|
+
CREATE INDEX IF NOT EXISTS idx_chats_updatedAt ON chats(updatedAt);
|
|
1900
|
+
|
|
1901
|
+
-- Messages table (nodes in the DAG)
|
|
1902
|
+
CREATE TABLE IF NOT EXISTS messages (
|
|
1903
|
+
id TEXT PRIMARY KEY,
|
|
1904
|
+
chatId TEXT NOT NULL,
|
|
1905
|
+
parentId TEXT,
|
|
1906
|
+
name TEXT NOT NULL,
|
|
1907
|
+
type TEXT,
|
|
1908
|
+
data TEXT NOT NULL,
|
|
1909
|
+
createdAt INTEGER NOT NULL,
|
|
1910
|
+
FOREIGN KEY (chatId) REFERENCES chats(id) ON DELETE CASCADE,
|
|
1911
|
+
FOREIGN KEY (parentId) REFERENCES messages(id)
|
|
1912
|
+
);
|
|
1913
|
+
|
|
1914
|
+
CREATE INDEX IF NOT EXISTS idx_messages_chatId ON messages(chatId);
|
|
1915
|
+
CREATE INDEX IF NOT EXISTS idx_messages_parentId ON messages(parentId);
|
|
1916
|
+
|
|
1917
|
+
-- Branches table (pointers to head messages)
|
|
1918
|
+
CREATE TABLE IF NOT EXISTS branches (
|
|
1919
|
+
id TEXT PRIMARY KEY,
|
|
1920
|
+
chatId TEXT NOT NULL,
|
|
1921
|
+
name TEXT NOT NULL,
|
|
1922
|
+
headMessageId TEXT,
|
|
1923
|
+
isActive INTEGER NOT NULL DEFAULT 0,
|
|
1924
|
+
createdAt INTEGER NOT NULL,
|
|
1925
|
+
FOREIGN KEY (chatId) REFERENCES chats(id) ON DELETE CASCADE,
|
|
1926
|
+
FOREIGN KEY (headMessageId) REFERENCES messages(id),
|
|
1927
|
+
UNIQUE(chatId, name)
|
|
1928
|
+
);
|
|
1929
|
+
|
|
1930
|
+
CREATE INDEX IF NOT EXISTS idx_branches_chatId ON branches(chatId);
|
|
1931
|
+
|
|
1932
|
+
-- Checkpoints table (pointers to message nodes)
|
|
1933
|
+
CREATE TABLE IF NOT EXISTS checkpoints (
|
|
1934
|
+
id TEXT PRIMARY KEY,
|
|
1935
|
+
chatId TEXT NOT NULL,
|
|
1936
|
+
name TEXT NOT NULL,
|
|
1937
|
+
messageId TEXT NOT NULL,
|
|
1938
|
+
createdAt INTEGER NOT NULL,
|
|
1939
|
+
FOREIGN KEY (chatId) REFERENCES chats(id) ON DELETE CASCADE,
|
|
1940
|
+
FOREIGN KEY (messageId) REFERENCES messages(id),
|
|
1941
|
+
UNIQUE(chatId, name)
|
|
1942
|
+
);
|
|
1943
|
+
|
|
1944
|
+
CREATE INDEX IF NOT EXISTS idx_checkpoints_chatId ON checkpoints(chatId);
|
|
1945
|
+
|
|
1946
|
+
-- FTS5 virtual table for full-text search
|
|
1947
|
+
-- messageId/chatId/name are UNINDEXED (stored but not searchable, used for filtering/joining)
|
|
1948
|
+
-- Only 'content' is indexed for full-text search
|
|
1949
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
|
|
1950
|
+
messageId UNINDEXED,
|
|
1951
|
+
chatId UNINDEXED,
|
|
1952
|
+
name UNINDEXED,
|
|
1953
|
+
content,
|
|
1954
|
+
tokenize='porter unicode61'
|
|
1955
|
+
);
|
|
1956
|
+
`;
|
|
1957
|
+
var SqliteContextStore = class extends ContextStore {
|
|
1958
|
+
#db;
|
|
1959
|
+
constructor(path3) {
|
|
1960
|
+
super();
|
|
1961
|
+
this.#db = new DatabaseSync(path3);
|
|
1962
|
+
this.#db.exec("PRAGMA foreign_keys = ON");
|
|
1963
|
+
this.#db.exec(STORE_DDL);
|
|
1964
|
+
}
|
|
1965
|
+
// ==========================================================================
|
|
1966
|
+
// Chat Operations
|
|
1967
|
+
// ==========================================================================
|
|
1968
|
+
async createChat(chat) {
|
|
1969
|
+
this.#db.prepare(
|
|
1970
|
+
`INSERT INTO chats (id, title, metadata)
|
|
1971
|
+
VALUES (?, ?, ?)`
|
|
1972
|
+
).run(
|
|
1973
|
+
chat.id,
|
|
1974
|
+
chat.title ?? null,
|
|
1975
|
+
chat.metadata ? JSON.stringify(chat.metadata) : null
|
|
1976
|
+
);
|
|
1977
|
+
}
|
|
1978
|
+
async upsertChat(chat) {
|
|
1979
|
+
const row = this.#db.prepare(
|
|
1980
|
+
`INSERT INTO chats (id, title, metadata)
|
|
1981
|
+
VALUES (?, ?, ?)
|
|
1982
|
+
ON CONFLICT(id) DO UPDATE SET id = excluded.id
|
|
1983
|
+
RETURNING *`
|
|
1984
|
+
).get(
|
|
1985
|
+
chat.id,
|
|
1986
|
+
chat.title ?? null,
|
|
1987
|
+
chat.metadata ? JSON.stringify(chat.metadata) : null
|
|
1988
|
+
);
|
|
1989
|
+
return {
|
|
1990
|
+
id: row.id,
|
|
1991
|
+
title: row.title ?? void 0,
|
|
1992
|
+
metadata: row.metadata ? JSON.parse(row.metadata) : void 0,
|
|
1993
|
+
createdAt: row.createdAt,
|
|
1994
|
+
updatedAt: row.updatedAt
|
|
1995
|
+
};
|
|
1996
|
+
}
|
|
1997
|
+
async getChat(chatId) {
|
|
1998
|
+
const row = this.#db.prepare("SELECT * FROM chats WHERE id = ?").get(chatId);
|
|
1999
|
+
if (!row) {
|
|
2000
|
+
return void 0;
|
|
1075
2001
|
}
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
2002
|
+
return {
|
|
2003
|
+
id: row.id,
|
|
2004
|
+
title: row.title ?? void 0,
|
|
2005
|
+
metadata: row.metadata ? JSON.parse(row.metadata) : void 0,
|
|
2006
|
+
createdAt: row.createdAt,
|
|
2007
|
+
updatedAt: row.updatedAt
|
|
2008
|
+
};
|
|
2009
|
+
}
|
|
2010
|
+
async updateChat(chatId, updates) {
|
|
2011
|
+
const setClauses = ["updatedAt = strftime('%s', 'now') * 1000"];
|
|
2012
|
+
const params = [];
|
|
2013
|
+
if (updates.title !== void 0) {
|
|
2014
|
+
setClauses.push("title = ?");
|
|
2015
|
+
params.push(updates.title ?? null);
|
|
1079
2016
|
}
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
}).filter((section) => Boolean(section));
|
|
1084
|
-
for (const [type, items] of grouped) {
|
|
1085
|
-
if (definedTypes.has(type)) {
|
|
1086
|
-
continue;
|
|
2017
|
+
if (updates.metadata !== void 0) {
|
|
2018
|
+
setClauses.push("metadata = ?");
|
|
2019
|
+
params.push(JSON.stringify(updates.metadata));
|
|
1087
2020
|
}
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
2021
|
+
params.push(chatId);
|
|
2022
|
+
const row = this.#db.prepare(
|
|
2023
|
+
`UPDATE chats SET ${setClauses.join(", ")} WHERE id = ? RETURNING *`
|
|
2024
|
+
).get(...params);
|
|
2025
|
+
return {
|
|
2026
|
+
id: row.id,
|
|
2027
|
+
title: row.title ?? void 0,
|
|
2028
|
+
metadata: row.metadata ? JSON.parse(row.metadata) : void 0,
|
|
2029
|
+
createdAt: row.createdAt,
|
|
2030
|
+
updatedAt: row.updatedAt
|
|
2031
|
+
};
|
|
2032
|
+
}
|
|
2033
|
+
async listChats() {
|
|
2034
|
+
const rows = this.#db.prepare(
|
|
2035
|
+
`SELECT
|
|
2036
|
+
c.id,
|
|
2037
|
+
c.title,
|
|
2038
|
+
c.createdAt,
|
|
2039
|
+
c.updatedAt,
|
|
2040
|
+
COUNT(DISTINCT m.id) as messageCount,
|
|
2041
|
+
COUNT(DISTINCT b.id) as branchCount
|
|
2042
|
+
FROM chats c
|
|
2043
|
+
LEFT JOIN messages m ON m.chatId = c.id
|
|
2044
|
+
LEFT JOIN branches b ON b.chatId = c.id
|
|
2045
|
+
GROUP BY c.id
|
|
2046
|
+
ORDER BY c.updatedAt DESC`
|
|
2047
|
+
).all();
|
|
2048
|
+
return rows.map((row) => ({
|
|
2049
|
+
id: row.id,
|
|
2050
|
+
title: row.title ?? void 0,
|
|
2051
|
+
messageCount: row.messageCount,
|
|
2052
|
+
branchCount: row.branchCount,
|
|
2053
|
+
createdAt: row.createdAt,
|
|
2054
|
+
updatedAt: row.updatedAt
|
|
2055
|
+
}));
|
|
2056
|
+
}
|
|
2057
|
+
// ==========================================================================
|
|
2058
|
+
// Message Operations (Graph Nodes)
|
|
2059
|
+
// ==========================================================================
|
|
2060
|
+
async addMessage(message2) {
|
|
2061
|
+
this.#db.prepare(
|
|
2062
|
+
`INSERT INTO messages (id, chatId, parentId, name, type, data, createdAt)
|
|
2063
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
2064
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
2065
|
+
parentId = excluded.parentId,
|
|
2066
|
+
name = excluded.name,
|
|
2067
|
+
type = excluded.type,
|
|
2068
|
+
data = excluded.data`
|
|
2069
|
+
).run(
|
|
2070
|
+
message2.id,
|
|
2071
|
+
message2.chatId,
|
|
2072
|
+
message2.parentId,
|
|
2073
|
+
message2.name,
|
|
2074
|
+
message2.type ?? null,
|
|
2075
|
+
JSON.stringify(message2.data),
|
|
2076
|
+
message2.createdAt
|
|
2077
|
+
);
|
|
2078
|
+
const content = typeof message2.data === "string" ? message2.data : JSON.stringify(message2.data);
|
|
2079
|
+
this.#db.prepare(`DELETE FROM messages_fts WHERE messageId = ?`).run(message2.id);
|
|
2080
|
+
this.#db.prepare(
|
|
2081
|
+
`INSERT INTO messages_fts(messageId, chatId, name, content)
|
|
2082
|
+
VALUES (?, ?, ?, ?)`
|
|
2083
|
+
).run(message2.id, message2.chatId, message2.name, content);
|
|
2084
|
+
}
|
|
2085
|
+
async getMessage(messageId) {
|
|
2086
|
+
const row = this.#db.prepare("SELECT * FROM messages WHERE id = ?").get(messageId);
|
|
2087
|
+
if (!row) {
|
|
2088
|
+
return void 0;
|
|
1091
2089
|
}
|
|
2090
|
+
return {
|
|
2091
|
+
id: row.id,
|
|
2092
|
+
chatId: row.chatId,
|
|
2093
|
+
parentId: row.parentId,
|
|
2094
|
+
name: row.name,
|
|
2095
|
+
type: row.type ?? void 0,
|
|
2096
|
+
data: JSON.parse(row.data),
|
|
2097
|
+
createdAt: row.createdAt
|
|
2098
|
+
};
|
|
2099
|
+
}
|
|
2100
|
+
async getMessageChain(headId) {
|
|
2101
|
+
const rows = this.#db.prepare(
|
|
2102
|
+
`WITH RECURSIVE chain AS (
|
|
2103
|
+
SELECT *, 0 as depth FROM messages WHERE id = ?
|
|
2104
|
+
UNION ALL
|
|
2105
|
+
SELECT m.*, c.depth + 1 FROM messages m
|
|
2106
|
+
INNER JOIN chain c ON m.id = c.parentId
|
|
2107
|
+
)
|
|
2108
|
+
SELECT * FROM chain
|
|
2109
|
+
ORDER BY depth DESC`
|
|
2110
|
+
).all(headId);
|
|
2111
|
+
return rows.map((row) => ({
|
|
2112
|
+
id: row.id,
|
|
2113
|
+
chatId: row.chatId,
|
|
2114
|
+
parentId: row.parentId,
|
|
2115
|
+
name: row.name,
|
|
2116
|
+
type: row.type ?? void 0,
|
|
2117
|
+
data: JSON.parse(row.data),
|
|
2118
|
+
createdAt: row.createdAt
|
|
2119
|
+
}));
|
|
2120
|
+
}
|
|
2121
|
+
async hasChildren(messageId) {
|
|
2122
|
+
const row = this.#db.prepare(
|
|
2123
|
+
"SELECT EXISTS(SELECT 1 FROM messages WHERE parentId = ?) as hasChildren"
|
|
2124
|
+
).get(messageId);
|
|
2125
|
+
return row.hasChildren === 1;
|
|
2126
|
+
}
|
|
2127
|
+
// ==========================================================================
|
|
2128
|
+
// Branch Operations
|
|
2129
|
+
// ==========================================================================
|
|
2130
|
+
async createBranch(branch) {
|
|
2131
|
+
this.#db.prepare(
|
|
2132
|
+
`INSERT INTO branches (id, chatId, name, headMessageId, isActive, createdAt)
|
|
2133
|
+
VALUES (?, ?, ?, ?, ?, ?)`
|
|
2134
|
+
).run(
|
|
2135
|
+
branch.id,
|
|
2136
|
+
branch.chatId,
|
|
2137
|
+
branch.name,
|
|
2138
|
+
branch.headMessageId,
|
|
2139
|
+
branch.isActive ? 1 : 0,
|
|
2140
|
+
branch.createdAt
|
|
2141
|
+
);
|
|
1092
2142
|
}
|
|
1093
|
-
|
|
1094
|
-
|
|
2143
|
+
async getBranch(chatId, name) {
|
|
2144
|
+
const row = this.#db.prepare("SELECT * FROM branches WHERE chatId = ? AND name = ?").get(chatId, name);
|
|
2145
|
+
if (!row) {
|
|
2146
|
+
return void 0;
|
|
2147
|
+
}
|
|
2148
|
+
return {
|
|
2149
|
+
id: row.id,
|
|
2150
|
+
chatId: row.chatId,
|
|
2151
|
+
name: row.name,
|
|
2152
|
+
headMessageId: row.headMessageId,
|
|
2153
|
+
isActive: row.isActive === 1,
|
|
2154
|
+
createdAt: row.createdAt
|
|
2155
|
+
};
|
|
2156
|
+
}
|
|
2157
|
+
async getActiveBranch(chatId) {
|
|
2158
|
+
const row = this.#db.prepare("SELECT * FROM branches WHERE chatId = ? AND isActive = 1").get(chatId);
|
|
2159
|
+
if (!row) {
|
|
2160
|
+
return void 0;
|
|
2161
|
+
}
|
|
2162
|
+
return {
|
|
2163
|
+
id: row.id,
|
|
2164
|
+
chatId: row.chatId,
|
|
2165
|
+
name: row.name,
|
|
2166
|
+
headMessageId: row.headMessageId,
|
|
2167
|
+
isActive: true,
|
|
2168
|
+
createdAt: row.createdAt
|
|
2169
|
+
};
|
|
2170
|
+
}
|
|
2171
|
+
async setActiveBranch(chatId, branchId) {
|
|
2172
|
+
this.#db.prepare("UPDATE branches SET isActive = 0 WHERE chatId = ?").run(chatId);
|
|
2173
|
+
this.#db.prepare("UPDATE branches SET isActive = 1 WHERE id = ?").run(branchId);
|
|
2174
|
+
}
|
|
2175
|
+
async updateBranchHead(branchId, messageId) {
|
|
2176
|
+
this.#db.prepare("UPDATE branches SET headMessageId = ? WHERE id = ?").run(messageId, branchId);
|
|
2177
|
+
}
|
|
2178
|
+
async listBranches(chatId) {
|
|
2179
|
+
const branches = this.#db.prepare(
|
|
2180
|
+
`SELECT
|
|
2181
|
+
b.id,
|
|
2182
|
+
b.name,
|
|
2183
|
+
b.headMessageId,
|
|
2184
|
+
b.isActive,
|
|
2185
|
+
b.createdAt
|
|
2186
|
+
FROM branches b
|
|
2187
|
+
WHERE b.chatId = ?
|
|
2188
|
+
ORDER BY b.createdAt ASC`
|
|
2189
|
+
).all(chatId);
|
|
2190
|
+
const result = [];
|
|
2191
|
+
for (const branch of branches) {
|
|
2192
|
+
let messageCount = 0;
|
|
2193
|
+
if (branch.headMessageId) {
|
|
2194
|
+
const countRow = this.#db.prepare(
|
|
2195
|
+
`WITH RECURSIVE chain AS (
|
|
2196
|
+
SELECT id, parentId FROM messages WHERE id = ?
|
|
2197
|
+
UNION ALL
|
|
2198
|
+
SELECT m.id, m.parentId FROM messages m
|
|
2199
|
+
INNER JOIN chain c ON m.id = c.parentId
|
|
2200
|
+
)
|
|
2201
|
+
SELECT COUNT(*) as count FROM chain`
|
|
2202
|
+
).get(branch.headMessageId);
|
|
2203
|
+
messageCount = countRow.count;
|
|
2204
|
+
}
|
|
2205
|
+
result.push({
|
|
2206
|
+
id: branch.id,
|
|
2207
|
+
name: branch.name,
|
|
2208
|
+
headMessageId: branch.headMessageId,
|
|
2209
|
+
isActive: branch.isActive === 1,
|
|
2210
|
+
messageCount,
|
|
2211
|
+
createdAt: branch.createdAt
|
|
2212
|
+
});
|
|
2213
|
+
}
|
|
2214
|
+
return result;
|
|
2215
|
+
}
|
|
2216
|
+
// ==========================================================================
|
|
2217
|
+
// Checkpoint Operations
|
|
2218
|
+
// ==========================================================================
|
|
2219
|
+
async createCheckpoint(checkpoint) {
|
|
2220
|
+
this.#db.prepare(
|
|
2221
|
+
`INSERT INTO checkpoints (id, chatId, name, messageId, createdAt)
|
|
2222
|
+
VALUES (?, ?, ?, ?, ?)
|
|
2223
|
+
ON CONFLICT(chatId, name) DO UPDATE SET
|
|
2224
|
+
messageId = excluded.messageId,
|
|
2225
|
+
createdAt = excluded.createdAt`
|
|
2226
|
+
).run(
|
|
2227
|
+
checkpoint.id,
|
|
2228
|
+
checkpoint.chatId,
|
|
2229
|
+
checkpoint.name,
|
|
2230
|
+
checkpoint.messageId,
|
|
2231
|
+
checkpoint.createdAt
|
|
2232
|
+
);
|
|
1095
2233
|
}
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
}
|
|
1101
|
-
var SECTION_ORDER = [
|
|
1102
|
-
// User context (render first - most important for personalization)
|
|
1103
|
-
{ type: "identity", tag: "identity" },
|
|
1104
|
-
{ type: "persona", tag: "persona" },
|
|
1105
|
-
{ type: "context", tag: "user_context" },
|
|
1106
|
-
{ type: "preference", tag: "user_preferences" },
|
|
1107
|
-
{ type: "alias", tag: "user_vocabulary" },
|
|
1108
|
-
{ type: "correction", tag: "user_corrections" },
|
|
1109
|
-
// Domain knowledge
|
|
1110
|
-
{ type: "guardrail", tag: "guardrails" },
|
|
1111
|
-
{ type: "styleGuide", tag: "style_guides" },
|
|
1112
|
-
{ type: "hint", tag: "hints" },
|
|
1113
|
-
{ type: "clarification", tag: "clarifications" },
|
|
1114
|
-
{ type: "workflow", tag: "workflows" },
|
|
1115
|
-
{ type: "quirk", tag: "quirks" },
|
|
1116
|
-
{ type: "term", tag: "terminology" },
|
|
1117
|
-
{ type: "explain", tag: "explanations" },
|
|
1118
|
-
{ type: "analogy", tag: "analogies" },
|
|
1119
|
-
{ type: "glossary", tag: "glossary" },
|
|
1120
|
-
{ type: "example", tag: "examples" }
|
|
1121
|
-
];
|
|
1122
|
-
function toTeachables(generated) {
|
|
1123
|
-
return generated.map((item) => {
|
|
1124
|
-
switch (item.type) {
|
|
1125
|
-
case "persona":
|
|
1126
|
-
return persona({ name: item.name, role: item.role, tone: item.tone });
|
|
1127
|
-
case "term":
|
|
1128
|
-
return term(item.name, item.definition);
|
|
1129
|
-
case "hint":
|
|
1130
|
-
return hint(item.text);
|
|
1131
|
-
case "guardrail":
|
|
1132
|
-
return guardrail({
|
|
1133
|
-
rule: item.rule,
|
|
1134
|
-
reason: item.reason,
|
|
1135
|
-
action: item.action
|
|
1136
|
-
});
|
|
1137
|
-
case "explain":
|
|
1138
|
-
return explain({
|
|
1139
|
-
concept: item.concept,
|
|
1140
|
-
explanation: item.explanation,
|
|
1141
|
-
therefore: item.therefore
|
|
1142
|
-
});
|
|
1143
|
-
case "example":
|
|
1144
|
-
return example({
|
|
1145
|
-
question: item.question,
|
|
1146
|
-
answer: item.answer,
|
|
1147
|
-
note: item.note
|
|
1148
|
-
});
|
|
1149
|
-
case "clarification":
|
|
1150
|
-
return clarification({
|
|
1151
|
-
when: item.when,
|
|
1152
|
-
ask: item.ask,
|
|
1153
|
-
reason: item.reason
|
|
1154
|
-
});
|
|
1155
|
-
case "workflow":
|
|
1156
|
-
return workflow({
|
|
1157
|
-
task: item.task,
|
|
1158
|
-
steps: item.steps,
|
|
1159
|
-
triggers: item.triggers,
|
|
1160
|
-
notes: item.notes
|
|
1161
|
-
});
|
|
1162
|
-
case "quirk":
|
|
1163
|
-
return quirk({
|
|
1164
|
-
issue: item.issue,
|
|
1165
|
-
workaround: item.workaround
|
|
1166
|
-
});
|
|
1167
|
-
case "styleGuide":
|
|
1168
|
-
return styleGuide({
|
|
1169
|
-
prefer: item.prefer,
|
|
1170
|
-
never: item.never,
|
|
1171
|
-
always: item.always
|
|
1172
|
-
});
|
|
1173
|
-
case "analogy":
|
|
1174
|
-
return analogy({
|
|
1175
|
-
concept: item.concept,
|
|
1176
|
-
relationship: item.relationship,
|
|
1177
|
-
insight: item.insight,
|
|
1178
|
-
therefore: item.therefore,
|
|
1179
|
-
pitfall: item.pitfall
|
|
1180
|
-
});
|
|
1181
|
-
case "glossary":
|
|
1182
|
-
return glossary(item.entries);
|
|
1183
|
-
// User-specific teachable types
|
|
1184
|
-
case "identity":
|
|
1185
|
-
return identity({ name: item.name, role: item.role });
|
|
1186
|
-
case "alias":
|
|
1187
|
-
return alias(item.term, item.meaning);
|
|
1188
|
-
case "preference":
|
|
1189
|
-
return preference(item.aspect, item.value);
|
|
1190
|
-
case "context":
|
|
1191
|
-
return context(item.description);
|
|
1192
|
-
case "correction":
|
|
1193
|
-
return correction(item.subject, item.clarification);
|
|
2234
|
+
async getCheckpoint(chatId, name) {
|
|
2235
|
+
const row = this.#db.prepare("SELECT * FROM checkpoints WHERE chatId = ? AND name = ?").get(chatId, name);
|
|
2236
|
+
if (!row) {
|
|
2237
|
+
return void 0;
|
|
1194
2238
|
}
|
|
1195
|
-
|
|
2239
|
+
return {
|
|
2240
|
+
id: row.id,
|
|
2241
|
+
chatId: row.chatId,
|
|
2242
|
+
name: row.name,
|
|
2243
|
+
messageId: row.messageId,
|
|
2244
|
+
createdAt: row.createdAt
|
|
2245
|
+
};
|
|
2246
|
+
}
|
|
2247
|
+
async listCheckpoints(chatId) {
|
|
2248
|
+
const rows = this.#db.prepare(
|
|
2249
|
+
`SELECT id, name, messageId, createdAt
|
|
2250
|
+
FROM checkpoints
|
|
2251
|
+
WHERE chatId = ?
|
|
2252
|
+
ORDER BY createdAt DESC`
|
|
2253
|
+
).all(chatId);
|
|
2254
|
+
return rows.map((row) => ({
|
|
2255
|
+
id: row.id,
|
|
2256
|
+
name: row.name,
|
|
2257
|
+
messageId: row.messageId,
|
|
2258
|
+
createdAt: row.createdAt
|
|
2259
|
+
}));
|
|
2260
|
+
}
|
|
2261
|
+
async deleteCheckpoint(chatId, name) {
|
|
2262
|
+
this.#db.prepare("DELETE FROM checkpoints WHERE chatId = ? AND name = ?").run(chatId, name);
|
|
2263
|
+
}
|
|
2264
|
+
// ==========================================================================
|
|
2265
|
+
// Search Operations
|
|
2266
|
+
// ==========================================================================
|
|
2267
|
+
async searchMessages(chatId, query, options) {
|
|
2268
|
+
const limit = options?.limit ?? 20;
|
|
2269
|
+
const roles = options?.roles;
|
|
2270
|
+
let sql = `
|
|
2271
|
+
SELECT
|
|
2272
|
+
m.id,
|
|
2273
|
+
m.chatId,
|
|
2274
|
+
m.parentId,
|
|
2275
|
+
m.name,
|
|
2276
|
+
m.type,
|
|
2277
|
+
m.data,
|
|
2278
|
+
m.createdAt,
|
|
2279
|
+
fts.rank,
|
|
2280
|
+
snippet(messages_fts, 3, '<mark>', '</mark>', '...', 32) as snippet
|
|
2281
|
+
FROM messages_fts fts
|
|
2282
|
+
JOIN messages m ON m.id = fts.messageId
|
|
2283
|
+
WHERE messages_fts MATCH ?
|
|
2284
|
+
AND fts.chatId = ?
|
|
2285
|
+
`;
|
|
2286
|
+
const params = [query, chatId];
|
|
2287
|
+
if (roles && roles.length > 0) {
|
|
2288
|
+
const placeholders = roles.map(() => "?").join(", ");
|
|
2289
|
+
sql += ` AND fts.name IN (${placeholders})`;
|
|
2290
|
+
params.push(...roles);
|
|
2291
|
+
}
|
|
2292
|
+
sql += " ORDER BY fts.rank LIMIT ?";
|
|
2293
|
+
params.push(limit);
|
|
2294
|
+
const rows = this.#db.prepare(sql).all(...params);
|
|
2295
|
+
return rows.map((row) => ({
|
|
2296
|
+
message: {
|
|
2297
|
+
id: row.id,
|
|
2298
|
+
chatId: row.chatId,
|
|
2299
|
+
parentId: row.parentId,
|
|
2300
|
+
name: row.name,
|
|
2301
|
+
type: row.type ?? void 0,
|
|
2302
|
+
data: JSON.parse(row.data),
|
|
2303
|
+
createdAt: row.createdAt
|
|
2304
|
+
},
|
|
2305
|
+
rank: row.rank,
|
|
2306
|
+
snippet: row.snippet
|
|
2307
|
+
}));
|
|
2308
|
+
}
|
|
2309
|
+
// ==========================================================================
|
|
2310
|
+
// Visualization Operations
|
|
2311
|
+
// ==========================================================================
|
|
2312
|
+
async getGraph(chatId) {
|
|
2313
|
+
const messageRows = this.#db.prepare(
|
|
2314
|
+
`SELECT id, parentId, name, data, createdAt
|
|
2315
|
+
FROM messages
|
|
2316
|
+
WHERE chatId = ?
|
|
2317
|
+
ORDER BY createdAt ASC`
|
|
2318
|
+
).all(chatId);
|
|
2319
|
+
const nodes = messageRows.map((row) => {
|
|
2320
|
+
const data = JSON.parse(row.data);
|
|
2321
|
+
const content = typeof data === "string" ? data : JSON.stringify(data);
|
|
2322
|
+
return {
|
|
2323
|
+
id: row.id,
|
|
2324
|
+
parentId: row.parentId,
|
|
2325
|
+
role: row.name,
|
|
2326
|
+
content: content.length > 50 ? content.slice(0, 50) + "..." : content,
|
|
2327
|
+
createdAt: row.createdAt
|
|
2328
|
+
};
|
|
2329
|
+
});
|
|
2330
|
+
const branchRows = this.#db.prepare(
|
|
2331
|
+
`SELECT name, headMessageId, isActive
|
|
2332
|
+
FROM branches
|
|
2333
|
+
WHERE chatId = ?
|
|
2334
|
+
ORDER BY createdAt ASC`
|
|
2335
|
+
).all(chatId);
|
|
2336
|
+
const branches = branchRows.map((row) => ({
|
|
2337
|
+
name: row.name,
|
|
2338
|
+
headMessageId: row.headMessageId,
|
|
2339
|
+
isActive: row.isActive === 1
|
|
2340
|
+
}));
|
|
2341
|
+
const checkpointRows = this.#db.prepare(
|
|
2342
|
+
`SELECT name, messageId
|
|
2343
|
+
FROM checkpoints
|
|
2344
|
+
WHERE chatId = ?
|
|
2345
|
+
ORDER BY createdAt ASC`
|
|
2346
|
+
).all(chatId);
|
|
2347
|
+
const checkpoints = checkpointRows.map((row) => ({
|
|
2348
|
+
name: row.name,
|
|
2349
|
+
messageId: row.messageId
|
|
2350
|
+
}));
|
|
2351
|
+
return {
|
|
2352
|
+
chatId,
|
|
2353
|
+
nodes,
|
|
2354
|
+
branches,
|
|
2355
|
+
checkpoints
|
|
2356
|
+
};
|
|
2357
|
+
}
|
|
2358
|
+
};
|
|
2359
|
+
var InMemoryContextStore = class extends SqliteContextStore {
|
|
2360
|
+
constructor() {
|
|
2361
|
+
super(":memory:");
|
|
2362
|
+
}
|
|
2363
|
+
};
|
|
2364
|
+
function structuredOutput(options) {
|
|
2365
|
+
return {
|
|
2366
|
+
async generate(contextVariables, config) {
|
|
2367
|
+
if (!options.context) {
|
|
2368
|
+
throw new Error(
|
|
2369
|
+
`structuredOutput "${options.name}" is missing a context.`
|
|
2370
|
+
);
|
|
2371
|
+
}
|
|
2372
|
+
if (!options.model) {
|
|
2373
|
+
throw new Error(
|
|
2374
|
+
`structuredOutput "${options.name}" is missing a model.`
|
|
2375
|
+
);
|
|
2376
|
+
}
|
|
2377
|
+
const { messages, systemPrompt } = await options.context.resolve({
|
|
2378
|
+
renderer: new XmlRenderer()
|
|
2379
|
+
});
|
|
2380
|
+
const result = await generateText({
|
|
2381
|
+
abortSignal: config?.abortSignal,
|
|
2382
|
+
providerOptions: options.providerOptions,
|
|
2383
|
+
model: options.model,
|
|
2384
|
+
system: systemPrompt,
|
|
2385
|
+
messages: convertToModelMessages(messages),
|
|
2386
|
+
stopWhen: stepCountIs(25),
|
|
2387
|
+
experimental_context: contextVariables,
|
|
2388
|
+
experimental_output: Output.object({ schema: options.schema })
|
|
2389
|
+
});
|
|
2390
|
+
return result.experimental_output;
|
|
2391
|
+
},
|
|
2392
|
+
async stream(contextVariables, config) {
|
|
2393
|
+
if (!options.context) {
|
|
2394
|
+
throw new Error(
|
|
2395
|
+
`structuredOutput "${options.name}" is missing a context.`
|
|
2396
|
+
);
|
|
2397
|
+
}
|
|
2398
|
+
if (!options.model) {
|
|
2399
|
+
throw new Error(
|
|
2400
|
+
`structuredOutput "${options.name}" is missing a model.`
|
|
2401
|
+
);
|
|
2402
|
+
}
|
|
2403
|
+
const { messages, systemPrompt } = await options.context.resolve({
|
|
2404
|
+
renderer: new XmlRenderer()
|
|
2405
|
+
});
|
|
2406
|
+
return streamText({
|
|
2407
|
+
abortSignal: config?.abortSignal,
|
|
2408
|
+
providerOptions: options.providerOptions,
|
|
2409
|
+
model: options.model,
|
|
2410
|
+
system: systemPrompt,
|
|
2411
|
+
messages: convertToModelMessages(messages),
|
|
2412
|
+
stopWhen: stepCountIs(25),
|
|
2413
|
+
experimental_transform: config?.transform ?? smoothStream(),
|
|
2414
|
+
experimental_context: contextVariables,
|
|
2415
|
+
experimental_output: Output.object({ schema: options.schema })
|
|
2416
|
+
});
|
|
2417
|
+
}
|
|
2418
|
+
};
|
|
1196
2419
|
}
|
|
1197
2420
|
|
|
1198
2421
|
// packages/text2sql/src/lib/agents/sql.agent.ts
|
|
@@ -1202,28 +2425,6 @@ var logger = new Console({
|
|
|
1202
2425
|
inspectOptions: { depth: null }
|
|
1203
2426
|
});
|
|
1204
2427
|
var RETRY_TEMPERATURES = [0, 0.2, 0.3];
|
|
1205
|
-
var sqlQueryAgent = agent5({
|
|
1206
|
-
name: "text2sql",
|
|
1207
|
-
model: groq5("openai/gpt-oss-20b"),
|
|
1208
|
-
logging: process.env.AGENT_LOGGING === "true",
|
|
1209
|
-
output: z5.union([
|
|
1210
|
-
z5.object({
|
|
1211
|
-
sql: z5.string().describe("The SQL query that answers the question"),
|
|
1212
|
-
reasoning: z5.string().optional().describe("The reasoning steps taken to generate the SQL")
|
|
1213
|
-
}),
|
|
1214
|
-
z5.object({
|
|
1215
|
-
error: z5.string().describe(
|
|
1216
|
-
"Error message explaining why the question cannot be answered with the given schema"
|
|
1217
|
-
)
|
|
1218
|
-
})
|
|
1219
|
-
]),
|
|
1220
|
-
prompt: (state) => {
|
|
1221
|
-
return `
|
|
1222
|
-
${state?.teachings || ""}
|
|
1223
|
-
${state?.introspection || ""}
|
|
1224
|
-
`;
|
|
1225
|
-
}
|
|
1226
|
-
});
|
|
1227
2428
|
function extractSql(output) {
|
|
1228
2429
|
const match = output.match(/```sql\n?([\s\S]*?)```/);
|
|
1229
2430
|
return match ? match[1].trim() : output.trim();
|
|
@@ -1231,8 +2432,8 @@ function extractSql(output) {
|
|
|
1231
2432
|
var marker = Symbol("SQLValidationError");
|
|
1232
2433
|
var SQLValidationError = class _SQLValidationError extends Error {
|
|
1233
2434
|
[marker];
|
|
1234
|
-
constructor(
|
|
1235
|
-
super(
|
|
2435
|
+
constructor(message2) {
|
|
2436
|
+
super(message2);
|
|
1236
2437
|
this.name = "SQLValidationError";
|
|
1237
2438
|
this[marker] = true;
|
|
1238
2439
|
}
|
|
@@ -1241,8 +2442,8 @@ var SQLValidationError = class _SQLValidationError extends Error {
|
|
|
1241
2442
|
}
|
|
1242
2443
|
};
|
|
1243
2444
|
var UnanswerableSQLError = class _UnanswerableSQLError extends Error {
|
|
1244
|
-
constructor(
|
|
1245
|
-
super(
|
|
2445
|
+
constructor(message2) {
|
|
2446
|
+
super(message2);
|
|
1246
2447
|
this.name = "UnanswerableSQLError";
|
|
1247
2448
|
}
|
|
1248
2449
|
static isInstance(error) {
|
|
@@ -1253,36 +2454,53 @@ async function toSql(options) {
|
|
|
1253
2454
|
const { maxRetries = 3 } = options;
|
|
1254
2455
|
return withRetry(
|
|
1255
2456
|
async (attemptNumber, errors, attempts) => {
|
|
1256
|
-
const
|
|
2457
|
+
const context = new ContextEngine({
|
|
2458
|
+
store: new InMemoryContextStore(),
|
|
2459
|
+
chatId: `sql-gen-${crypto.randomUUID()}`
|
|
2460
|
+
});
|
|
2461
|
+
context.set(
|
|
2462
|
+
persona({
|
|
2463
|
+
name: "Freya",
|
|
2464
|
+
role: "You are an expert SQL query generator. You translate natural language questions into precise, efficient SQL queries based on the provided database schema."
|
|
2465
|
+
}),
|
|
2466
|
+
...options.instructions,
|
|
2467
|
+
...options.schemaFragments
|
|
2468
|
+
);
|
|
2469
|
+
if (errors.length) {
|
|
2470
|
+
context.set(
|
|
2471
|
+
user6(options.input),
|
|
2472
|
+
user6(
|
|
2473
|
+
`<validation_error>Your previous SQL query had the following error: ${errors.at(-1)?.message}. Please fix the query.</validation_error>`
|
|
2474
|
+
)
|
|
2475
|
+
);
|
|
2476
|
+
} else {
|
|
2477
|
+
context.set(user6(options.input));
|
|
2478
|
+
}
|
|
2479
|
+
const sqlOutput = structuredOutput({
|
|
2480
|
+
name: "text2sql",
|
|
1257
2481
|
model: wrapLanguageModel2({
|
|
1258
|
-
model: options.model ??
|
|
2482
|
+
model: options.model ?? groq5("openai/gpt-oss-20b"),
|
|
1259
2483
|
middleware: defaultSettingsMiddleware2({
|
|
1260
2484
|
settings: {
|
|
1261
2485
|
temperature: RETRY_TEMPERATURES[attemptNumber - 1] ?? 0.3,
|
|
1262
2486
|
topP: 1
|
|
1263
2487
|
}
|
|
1264
2488
|
})
|
|
1265
|
-
})
|
|
2489
|
+
}),
|
|
2490
|
+
context,
|
|
2491
|
+
schema: z5.union([
|
|
2492
|
+
z5.object({
|
|
2493
|
+
sql: z5.string().describe("The SQL query that answers the question"),
|
|
2494
|
+
reasoning: z5.string().optional().describe("The reasoning steps taken to generate the SQL")
|
|
2495
|
+
}),
|
|
2496
|
+
z5.object({
|
|
2497
|
+
error: z5.string().describe(
|
|
2498
|
+
"Error message explaining why the question cannot be answered with the given schema"
|
|
2499
|
+
)
|
|
2500
|
+
})
|
|
2501
|
+
])
|
|
1266
2502
|
});
|
|
1267
|
-
const
|
|
1268
|
-
user6(options.input),
|
|
1269
|
-
user6(
|
|
1270
|
-
`<validation_error>Your previous SQL query had the following error: ${errors.at(-1)?.message}. Please fix the query.</validation_error>`
|
|
1271
|
-
)
|
|
1272
|
-
] : [user6(options.input)];
|
|
1273
|
-
const output = await toOutput(
|
|
1274
|
-
generate6(agentInstance, messages, {
|
|
1275
|
-
introspection: options.introspection,
|
|
1276
|
-
teachings: toInstructions(
|
|
1277
|
-
"instructions",
|
|
1278
|
-
persona({
|
|
1279
|
-
name: "Freya",
|
|
1280
|
-
role: "You are an expert SQL query generator. You translate natural language questions into precise, efficient SQL queries based on the provided database schema."
|
|
1281
|
-
}),
|
|
1282
|
-
...options.instructions
|
|
1283
|
-
)
|
|
1284
|
-
})
|
|
1285
|
-
);
|
|
2503
|
+
const output = await sqlOutput.generate();
|
|
1286
2504
|
if ("error" in output) {
|
|
1287
2505
|
throw new UnanswerableSQLError(output.error);
|
|
1288
2506
|
}
|
|
@@ -1321,35 +2539,35 @@ async function withRetry(computation, options = { retries: 3 }) {
|
|
|
1321
2539
|
},
|
|
1322
2540
|
{
|
|
1323
2541
|
retries: options.retries,
|
|
1324
|
-
shouldRetry: (
|
|
1325
|
-
if (UnanswerableSQLError.isInstance(
|
|
2542
|
+
shouldRetry: (context) => {
|
|
2543
|
+
if (UnanswerableSQLError.isInstance(context.error)) {
|
|
1326
2544
|
return false;
|
|
1327
2545
|
}
|
|
1328
|
-
if (SQLValidationError.isInstance(
|
|
2546
|
+
if (SQLValidationError.isInstance(context.error)) {
|
|
1329
2547
|
return true;
|
|
1330
2548
|
}
|
|
1331
2549
|
console.log({
|
|
1332
2550
|
NoObjectGeneratedError: NoObjectGeneratedError.isInstance(
|
|
1333
|
-
|
|
2551
|
+
context.error
|
|
1334
2552
|
),
|
|
1335
2553
|
NoOutputGeneratedError: NoOutputGeneratedError.isInstance(
|
|
1336
|
-
|
|
2554
|
+
context.error
|
|
1337
2555
|
),
|
|
1338
|
-
APICallError: APICallError.isInstance(
|
|
1339
|
-
JSONParseError: JSONParseError.isInstance(
|
|
1340
|
-
TypeValidationError: TypeValidationError.isInstance(
|
|
2556
|
+
APICallError: APICallError.isInstance(context.error),
|
|
2557
|
+
JSONParseError: JSONParseError.isInstance(context.error),
|
|
2558
|
+
TypeValidationError: TypeValidationError.isInstance(context.error),
|
|
1341
2559
|
NoContentGeneratedError: NoContentGeneratedError.isInstance(
|
|
1342
|
-
|
|
2560
|
+
context.error
|
|
1343
2561
|
)
|
|
1344
2562
|
});
|
|
1345
|
-
return APICallError.isInstance(
|
|
2563
|
+
return APICallError.isInstance(context.error) || JSONParseError.isInstance(context.error) || TypeValidationError.isInstance(context.error) || NoObjectGeneratedError.isInstance(context.error) || NoOutputGeneratedError.isInstance(context.error) || NoContentGeneratedError.isInstance(context.error);
|
|
1346
2564
|
},
|
|
1347
|
-
onFailedAttempt(
|
|
1348
|
-
logger.error(`toSQL`,
|
|
2565
|
+
onFailedAttempt(context) {
|
|
2566
|
+
logger.error(`toSQL`, context.error);
|
|
1349
2567
|
console.log(
|
|
1350
|
-
`Attempt ${
|
|
2568
|
+
`Attempt ${context.attemptNumber} failed. There are ${context.retriesLeft} retries left.`
|
|
1351
2569
|
);
|
|
1352
|
-
errors.push(
|
|
2570
|
+
errors.push(context.error);
|
|
1353
2571
|
}
|
|
1354
2572
|
}
|
|
1355
2573
|
);
|
|
@@ -1379,7 +2597,7 @@ var SchemaSynthesizer = class extends PairProducer {
|
|
|
1379
2597
|
* @returns Generated pairs from all combinations
|
|
1380
2598
|
*/
|
|
1381
2599
|
async *produce() {
|
|
1382
|
-
const introspection =
|
|
2600
|
+
const introspection = "";
|
|
1383
2601
|
const combinations = this.#personas.flatMap(
|
|
1384
2602
|
(persona2) => this.#complexities.map((complexity) => ({ persona: persona2, complexity }))
|
|
1385
2603
|
);
|
|
@@ -1421,7 +2639,8 @@ Generate ${this.options.count} questions at ${complexity} complexity.` : void 0;
|
|
|
1421
2639
|
return await toSql({
|
|
1422
2640
|
input: question,
|
|
1423
2641
|
adapter: this.adapter,
|
|
1424
|
-
|
|
2642
|
+
schemaFragments: [],
|
|
2643
|
+
// Placeholder - needs to pass actual fragments
|
|
1425
2644
|
instructions: this.options.teachings ?? [],
|
|
1426
2645
|
model: this.options.model
|
|
1427
2646
|
});
|
|
@@ -1456,9 +2675,9 @@ import dedent5 from "dedent";
|
|
|
1456
2675
|
import pLimit2 from "p-limit";
|
|
1457
2676
|
import z6 from "zod";
|
|
1458
2677
|
import {
|
|
1459
|
-
agent as
|
|
1460
|
-
generate as
|
|
1461
|
-
toOutput
|
|
2678
|
+
agent as agent5,
|
|
2679
|
+
generate as generate6,
|
|
2680
|
+
toOutput,
|
|
1462
2681
|
user as user7
|
|
1463
2682
|
} from "@deepagents/agent";
|
|
1464
2683
|
|
|
@@ -1496,7 +2715,7 @@ var styleInstructions = {
|
|
|
1496
2715
|
};
|
|
1497
2716
|
|
|
1498
2717
|
// packages/text2sql/src/lib/synthesis/synthesizers/breadth-evolver.ts
|
|
1499
|
-
var paraphraserAgent =
|
|
2718
|
+
var paraphraserAgent = agent5({
|
|
1500
2719
|
name: "question_paraphraser",
|
|
1501
2720
|
model: wrapLanguageModel3({
|
|
1502
2721
|
model: groq6("openai/gpt-oss-20b"),
|
|
@@ -1592,8 +2811,8 @@ var BreadthEvolver = class extends PairProducer {
|
|
|
1592
2811
|
for await (const chunk of this.from(this.source)) {
|
|
1593
2812
|
const tasks = chunk.map(
|
|
1594
2813
|
(pair) => this.#limit(async () => {
|
|
1595
|
-
const { paraphrases } = await
|
|
1596
|
-
|
|
2814
|
+
const { paraphrases } = await toOutput(
|
|
2815
|
+
generate6(
|
|
1597
2816
|
paraphraserAgent.clone({ model: this.options.model }),
|
|
1598
2817
|
[
|
|
1599
2818
|
user7(
|
|
@@ -1634,7 +2853,7 @@ import dedent6 from "dedent";
|
|
|
1634
2853
|
import pLimit3 from "p-limit";
|
|
1635
2854
|
import pRetry2 from "p-retry";
|
|
1636
2855
|
import z7 from "zod";
|
|
1637
|
-
import { agent as
|
|
2856
|
+
import { agent as agent6, generate as generate7, user as user8 } from "@deepagents/agent";
|
|
1638
2857
|
var techniqueInstructions = {
|
|
1639
2858
|
"add-aggregation": dedent6`
|
|
1640
2859
|
Add aggregation requirements to the question.
|
|
@@ -1677,7 +2896,7 @@ var techniqueInstructions = {
|
|
|
1677
2896
|
- "Get costs" → "What would be the impact of a 10% discount on profit margins?"
|
|
1678
2897
|
`
|
|
1679
2898
|
};
|
|
1680
|
-
var questionEvolverAgent =
|
|
2899
|
+
var questionEvolverAgent = agent6({
|
|
1681
2900
|
name: "question_evolver",
|
|
1682
2901
|
model: wrapLanguageModel4({
|
|
1683
2902
|
model: groq7("openai/gpt-oss-20b"),
|
|
@@ -1760,7 +2979,7 @@ var DepthEvolver = class extends PairProducer {
|
|
|
1760
2979
|
* Removes batch barrier - no longer waits for all evolutions before yielding.
|
|
1761
2980
|
*/
|
|
1762
2981
|
async *produce() {
|
|
1763
|
-
const introspection =
|
|
2982
|
+
const introspection = "";
|
|
1764
2983
|
const count = this.options?.count ?? 1;
|
|
1765
2984
|
const techniques = this.options?.techniques ?? ALL_TECHNIQUES;
|
|
1766
2985
|
let pairIndex = 0;
|
|
@@ -1780,7 +2999,7 @@ var DepthEvolver = class extends PairProducer {
|
|
|
1780
2999
|
}
|
|
1781
3000
|
async #processTask(pair, technique, introspection) {
|
|
1782
3001
|
const { experimental_output } = await withRetry2(
|
|
1783
|
-
() =>
|
|
3002
|
+
() => generate7(
|
|
1784
3003
|
questionEvolverAgent.clone({
|
|
1785
3004
|
model: this.options?.model
|
|
1786
3005
|
}),
|
|
@@ -1799,7 +3018,8 @@ var DepthEvolver = class extends PairProducer {
|
|
|
1799
3018
|
const sqlResult = await toSql({
|
|
1800
3019
|
input: evolvedQuestion,
|
|
1801
3020
|
adapter: this.adapter,
|
|
1802
|
-
|
|
3021
|
+
schemaFragments: [],
|
|
3022
|
+
// Placeholder - needs to pass actual fragments
|
|
1803
3023
|
instructions: [],
|
|
1804
3024
|
model: this.options?.model
|
|
1805
3025
|
});
|
|
@@ -1828,22 +3048,22 @@ var DepthEvolver = class extends PairProducer {
|
|
|
1828
3048
|
async function withRetry2(computation) {
|
|
1829
3049
|
return pRetry2(computation, {
|
|
1830
3050
|
retries: 3,
|
|
1831
|
-
shouldRetry: (
|
|
3051
|
+
shouldRetry: (context) => {
|
|
1832
3052
|
console.log({
|
|
1833
3053
|
NoObjectGeneratedError: NoObjectGeneratedError2.isInstance(
|
|
1834
|
-
|
|
3054
|
+
context.error
|
|
1835
3055
|
),
|
|
1836
3056
|
NoOutputGeneratedError: NoOutputGeneratedError2.isInstance(
|
|
1837
|
-
|
|
3057
|
+
context.error
|
|
1838
3058
|
)
|
|
1839
3059
|
});
|
|
1840
|
-
return NoObjectGeneratedError2.isInstance(
|
|
3060
|
+
return NoObjectGeneratedError2.isInstance(context.error) || NoOutputGeneratedError2.isInstance(context.error);
|
|
1841
3061
|
},
|
|
1842
|
-
onFailedAttempt(
|
|
3062
|
+
onFailedAttempt(context) {
|
|
1843
3063
|
console.log(
|
|
1844
|
-
`Attempt ${
|
|
3064
|
+
`Attempt ${context.attemptNumber} failed. There are ${context.retriesLeft} retries left.`
|
|
1845
3065
|
);
|
|
1846
|
-
console.dir(
|
|
3066
|
+
console.dir(context.error, { depth: null });
|
|
1847
3067
|
}
|
|
1848
3068
|
});
|
|
1849
3069
|
}
|
|
@@ -1853,70 +3073,65 @@ import { groq as groq8 } from "@ai-sdk/groq";
|
|
|
1853
3073
|
import { defaultSettingsMiddleware as defaultSettingsMiddleware5, wrapLanguageModel as wrapLanguageModel5 } from "ai";
|
|
1854
3074
|
import dedent7 from "dedent";
|
|
1855
3075
|
import z8 from "zod";
|
|
1856
|
-
import
|
|
1857
|
-
var
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
3076
|
+
import "@deepagents/agent";
|
|
3077
|
+
var outputSchema = z8.object({
|
|
3078
|
+
personas: z8.array(
|
|
3079
|
+
z8.object({
|
|
3080
|
+
role: z8.string().describe("The job title or role of this persona"),
|
|
3081
|
+
perspective: z8.string().describe(
|
|
3082
|
+
"Rich description of what this persona cares about when querying the database"
|
|
3083
|
+
),
|
|
3084
|
+
styles: z8.array(z8.enum(ALL_STYLES)).min(1).max(3).describe(
|
|
3085
|
+
"Typical communication styles for this persona (1-3 styles)"
|
|
3086
|
+
)
|
|
1863
3087
|
})
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
)
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
<identity>
|
|
1882
|
-
You are an expert at understanding database schemas and inferring who would use them.
|
|
1883
|
-
Your task is to analyze a database schema and generate realistic personas representing
|
|
1884
|
-
the different types of users who would query this database.
|
|
1885
|
-
</identity>
|
|
3088
|
+
).min(1).describe("List of personas who would query this database")
|
|
3089
|
+
});
|
|
3090
|
+
async function generatePersonas(schemaFragments, options) {
|
|
3091
|
+
const schema = new XmlRenderer().render(schemaFragments);
|
|
3092
|
+
const count = options?.count ?? 5;
|
|
3093
|
+
const context = new ContextEngine({
|
|
3094
|
+
store: new InMemoryContextStore(),
|
|
3095
|
+
chatId: `persona-gen-${crypto.randomUUID()}`
|
|
3096
|
+
});
|
|
3097
|
+
context.set(
|
|
3098
|
+
persona({
|
|
3099
|
+
name: "persona_generator",
|
|
3100
|
+
role: "You are an expert at understanding database schemas and inferring who would use them."
|
|
3101
|
+
}),
|
|
3102
|
+
role(dedent7`
|
|
3103
|
+
Your task is to analyze a database schema and generate realistic personas representing
|
|
3104
|
+
the different types of users who would query this database.
|
|
1886
3105
|
|
|
1887
3106
|
<database_schema>
|
|
1888
|
-
${
|
|
3107
|
+
${schema}
|
|
1889
3108
|
</database_schema>
|
|
1890
3109
|
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
- Perspectives should be detailed enough to guide question paraphrasing
|
|
1917
|
-
- Cover different levels of technical expertise (some technical, some business-focused)
|
|
1918
|
-
- Styles should match how this persona would naturally communicate
|
|
1919
|
-
</task>
|
|
3110
|
+
For each persona, provide:
|
|
3111
|
+
1. **role**: Their job title or role (e.g., "Financial Analyst", "Customer Support Rep")
|
|
3112
|
+
2. **perspective**: A rich description of what they care about, including:
|
|
3113
|
+
- What questions they typically ask
|
|
3114
|
+
- What metrics/data points matter to them
|
|
3115
|
+
- How they prefer data formatted or presented
|
|
3116
|
+
- Their priorities (speed vs accuracy, detail vs summary)
|
|
3117
|
+
- Domain-specific concerns relevant to their role
|
|
3118
|
+
3. **styles**: 1-3 communication styles typical for this persona. Choose from:
|
|
3119
|
+
- formal: Professional business language, complete sentences
|
|
3120
|
+
- colloquial: Casual everyday speech, contractions
|
|
3121
|
+
- imperative: Commands like "Show me...", "Get...", "List..."
|
|
3122
|
+
- interrogative: Questions like "What is...", "How many..."
|
|
3123
|
+
- descriptive: Verbose, detailed phrasing
|
|
3124
|
+
- concise: Brief, minimal words
|
|
3125
|
+
- vague: Ambiguous, hedging language
|
|
3126
|
+
- metaphorical: Figurative language, analogies
|
|
3127
|
+
- conversational: Chat-like, casual tone
|
|
3128
|
+
|
|
3129
|
+
Requirements:
|
|
3130
|
+
- Personas should be realistic for the given schema
|
|
3131
|
+
- Each persona should have distinct concerns and priorities
|
|
3132
|
+
- Perspectives should be detailed enough to guide question paraphrasing
|
|
3133
|
+
- Cover different levels of technical expertise (some technical, some business-focused)
|
|
3134
|
+
- Styles should match how this persona would naturally communicate
|
|
1920
3135
|
|
|
1921
3136
|
<example>
|
|
1922
3137
|
For an e-commerce schema with orders, customers, products tables:
|
|
@@ -1939,46 +3154,33 @@ var personaGeneratorAgent = agent8({
|
|
|
1939
3154
|
- Do not invent tables or data that don't exist in the schema
|
|
1940
3155
|
- Ensure perspectives are specific to the domain, not generic
|
|
1941
3156
|
</guardrails>
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
}
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
const { experimental_output } = await generate9(
|
|
1962
|
-
personaGeneratorAgent.clone({
|
|
1963
|
-
model: this.options?.model
|
|
1964
|
-
}),
|
|
1965
|
-
[user9(`Generate ${count} personas for this database schema.`)],
|
|
1966
|
-
{
|
|
1967
|
-
schema,
|
|
1968
|
-
count
|
|
1969
|
-
}
|
|
1970
|
-
);
|
|
1971
|
-
return experimental_output.personas;
|
|
1972
|
-
}
|
|
1973
|
-
};
|
|
3157
|
+
`),
|
|
3158
|
+
user6(
|
|
3159
|
+
`Generate exactly ${count} distinct personas who would query this database.`
|
|
3160
|
+
)
|
|
3161
|
+
);
|
|
3162
|
+
const personaOutput = structuredOutput({
|
|
3163
|
+
name: "persona_generator",
|
|
3164
|
+
model: wrapLanguageModel5({
|
|
3165
|
+
model: options?.model ?? groq8("openai/gpt-oss-20b"),
|
|
3166
|
+
middleware: defaultSettingsMiddleware5({
|
|
3167
|
+
settings: { temperature: 0.8, topP: 0.95, presencePenalty: 0.2 }
|
|
3168
|
+
})
|
|
3169
|
+
}),
|
|
3170
|
+
context,
|
|
3171
|
+
schema: outputSchema
|
|
3172
|
+
});
|
|
3173
|
+
const output = await personaOutput.generate();
|
|
3174
|
+
return output.personas;
|
|
3175
|
+
}
|
|
1974
3176
|
|
|
1975
3177
|
// packages/text2sql/src/lib/agents/teachables.agent.ts
|
|
1976
3178
|
import { groq as groq9 } from "@ai-sdk/groq";
|
|
1977
3179
|
import { defaultSettingsMiddleware as defaultSettingsMiddleware6, wrapLanguageModel as wrapLanguageModel6 } from "ai";
|
|
1978
3180
|
import dedent8 from "dedent";
|
|
1979
3181
|
import z9 from "zod";
|
|
1980
|
-
import { agent as
|
|
1981
|
-
var
|
|
3182
|
+
import { agent as agent7, generate as generate8, user as user9 } from "@deepagents/agent";
|
|
3183
|
+
var outputSchema2 = z9.object({
|
|
1982
3184
|
terms: z9.array(z9.object({ name: z9.string(), definition: z9.string() })).optional().describe("Domain terminology definitions"),
|
|
1983
3185
|
hints: z9.array(z9.object({ text: z9.string() })).optional().describe("Helpful hints for SQL generation"),
|
|
1984
3186
|
guardrails: z9.array(
|
|
@@ -2021,7 +3223,7 @@ var outputSchema = z9.object({
|
|
|
2021
3223
|
).optional().describe("SQL style preferences"),
|
|
2022
3224
|
analogies: z9.array(
|
|
2023
3225
|
z9.object({
|
|
2024
|
-
|
|
3226
|
+
concepts: z9.array(z9.string()).min(2),
|
|
2025
3227
|
relationship: z9.string(),
|
|
2026
3228
|
insight: z9.string().optional(),
|
|
2027
3229
|
therefore: z9.string().optional(),
|
|
@@ -2029,7 +3231,7 @@ var outputSchema = z9.object({
|
|
|
2029
3231
|
})
|
|
2030
3232
|
).optional().describe("Concept analogies")
|
|
2031
3233
|
});
|
|
2032
|
-
var teachablesAuthorAgent =
|
|
3234
|
+
var teachablesAuthorAgent = agent7({
|
|
2033
3235
|
name: "teachables-author",
|
|
2034
3236
|
model: wrapLanguageModel6({
|
|
2035
3237
|
model: groq9("openai/gpt-oss-20b"),
|
|
@@ -2037,10 +3239,10 @@ var teachablesAuthorAgent = agent9({
|
|
|
2037
3239
|
settings: { temperature: 0.4, topP: 0.95 }
|
|
2038
3240
|
})
|
|
2039
3241
|
}),
|
|
2040
|
-
output:
|
|
3242
|
+
output: outputSchema2,
|
|
2041
3243
|
prompt: (state) => dedent8`
|
|
2042
3244
|
<identity>
|
|
2043
|
-
You design "
|
|
3245
|
+
You design "fragments" for a Text2SQL system. Fragments become structured XML instructions.
|
|
2044
3246
|
Choose only high-impact items that improve accuracy, safety, or clarity for this database.
|
|
2045
3247
|
</identity>
|
|
2046
3248
|
|
|
@@ -2061,12 +3263,12 @@ var teachablesAuthorAgent = agent9({
|
|
|
2061
3263
|
- workflows: [{ task: string, steps: string[], triggers?: string[], notes?: string }] - Multi-step tasks
|
|
2062
3264
|
- quirks: [{ issue: string, workaround: string }] - Known issues
|
|
2063
3265
|
- styleGuides: [{ prefer: string, never?: string, always?: string }] - SQL style rules
|
|
2064
|
-
- analogies: [{
|
|
3266
|
+
- analogies: [{ concepts: string[], relationship: string, insight?: string, therefore?: string, pitfall?: string }]
|
|
2065
3267
|
</output_structure>
|
|
2066
3268
|
|
|
2067
3269
|
<instructions>
|
|
2068
3270
|
1. Analyze the schema to infer domain, relationships, and sensitive columns.
|
|
2069
|
-
2. Generate 3-10
|
|
3271
|
+
2. Generate 3-10 fragments total across all categories, prioritizing:
|
|
2070
3272
|
- guardrails for PII columns (email, ssn, phone, etc)
|
|
2071
3273
|
- hints for status/enum columns
|
|
2072
3274
|
- clarifications for ambiguous terms
|
|
@@ -2076,75 +3278,95 @@ var teachablesAuthorAgent = agent9({
|
|
|
2076
3278
|
`
|
|
2077
3279
|
});
|
|
2078
3280
|
async function toTeachings(input, options) {
|
|
2079
|
-
const { experimental_output: result } = await
|
|
3281
|
+
const { experimental_output: result } = await generate8(
|
|
2080
3282
|
teachablesAuthorAgent.clone({ model: options?.model }),
|
|
2081
3283
|
[
|
|
2082
|
-
|
|
2083
|
-
`Analyze this database schema and generate
|
|
3284
|
+
user9(
|
|
3285
|
+
`Analyze this database schema and generate fragments that will help an AI generate accurate SQL queries.`
|
|
2084
3286
|
)
|
|
2085
3287
|
],
|
|
2086
3288
|
input
|
|
2087
3289
|
);
|
|
2088
|
-
const
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
3290
|
+
const fragments = [];
|
|
3291
|
+
result.terms?.forEach((t) => fragments.push(term(t.name, t.definition)));
|
|
3292
|
+
result.hints?.forEach((h) => fragments.push(hint(h.text)));
|
|
3293
|
+
result.guardrails?.forEach(
|
|
3294
|
+
(g) => fragments.push(
|
|
3295
|
+
guardrail({ rule: g.rule, reason: g.reason, action: g.action })
|
|
3296
|
+
)
|
|
3297
|
+
);
|
|
3298
|
+
result.explains?.forEach(
|
|
3299
|
+
(e) => fragments.push(
|
|
3300
|
+
explain({
|
|
3301
|
+
concept: e.concept,
|
|
3302
|
+
explanation: e.explanation,
|
|
3303
|
+
therefore: e.therefore
|
|
3304
|
+
})
|
|
3305
|
+
)
|
|
3306
|
+
);
|
|
3307
|
+
result.examples?.forEach(
|
|
3308
|
+
(e) => fragments.push(
|
|
3309
|
+
example({ question: e.question, answer: e.answer, note: e.note })
|
|
3310
|
+
)
|
|
3311
|
+
);
|
|
3312
|
+
result.clarifications?.forEach(
|
|
3313
|
+
(c) => fragments.push(
|
|
3314
|
+
clarification({ when: c.when, ask: c.ask, reason: c.reason })
|
|
3315
|
+
)
|
|
3316
|
+
);
|
|
3317
|
+
result.workflows?.forEach(
|
|
3318
|
+
(w) => fragments.push(
|
|
3319
|
+
workflow({
|
|
3320
|
+
task: w.task,
|
|
3321
|
+
steps: w.steps,
|
|
3322
|
+
triggers: w.triggers,
|
|
3323
|
+
notes: w.notes
|
|
3324
|
+
})
|
|
3325
|
+
)
|
|
3326
|
+
);
|
|
3327
|
+
result.quirks?.forEach(
|
|
3328
|
+
(q) => fragments.push(quirk({ issue: q.issue, workaround: q.workaround }))
|
|
3329
|
+
);
|
|
3330
|
+
result.styleGuides?.forEach(
|
|
3331
|
+
(s) => fragments.push(
|
|
3332
|
+
styleGuide({ prefer: s.prefer, never: s.never, always: s.always })
|
|
3333
|
+
)
|
|
3334
|
+
);
|
|
3335
|
+
result.analogies?.forEach(
|
|
3336
|
+
(a) => fragments.push(
|
|
3337
|
+
analogy({
|
|
3338
|
+
concepts: a.concepts,
|
|
3339
|
+
relationship: a.relationship,
|
|
3340
|
+
insight: a.insight,
|
|
3341
|
+
therefore: a.therefore,
|
|
3342
|
+
pitfall: a.pitfall
|
|
3343
|
+
})
|
|
3344
|
+
)
|
|
3345
|
+
);
|
|
3346
|
+
return fragments;
|
|
2107
3347
|
}
|
|
2108
3348
|
|
|
2109
3349
|
// packages/text2sql/src/lib/synthesis/synthesizers/teachings-generator.ts
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
async generate(maxRetries = 3) {
|
|
2126
|
-
const schema = await this.adapter.introspect();
|
|
2127
|
-
let lastError;
|
|
2128
|
-
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
2129
|
-
try {
|
|
2130
|
-
return await toTeachings(
|
|
2131
|
-
{
|
|
2132
|
-
schema,
|
|
2133
|
-
context: this.options?.context
|
|
2134
|
-
},
|
|
2135
|
-
{ model: this.options?.model }
|
|
2136
|
-
);
|
|
2137
|
-
} catch (error) {
|
|
2138
|
-
lastError = error;
|
|
2139
|
-
const isRetryable = lastError.message.includes("parse") || lastError.message.includes("schema") || lastError.message.includes("No object generated") || lastError.name.includes("AI_");
|
|
2140
|
-
if (!isRetryable) {
|
|
2141
|
-
throw lastError;
|
|
2142
|
-
}
|
|
3350
|
+
async function generateTeachings(schemaFragments, options) {
|
|
3351
|
+
const schema = new XmlRenderer().render(schemaFragments);
|
|
3352
|
+
const maxRetries = options?.maxRetries ?? 3;
|
|
3353
|
+
let lastError;
|
|
3354
|
+
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
3355
|
+
try {
|
|
3356
|
+
return await toTeachings(
|
|
3357
|
+
{ schema, context: options?.context },
|
|
3358
|
+
{ model: options?.model }
|
|
3359
|
+
);
|
|
3360
|
+
} catch (error) {
|
|
3361
|
+
lastError = error;
|
|
3362
|
+
const isRetryable = lastError.message.includes("parse") || lastError.message.includes("schema") || lastError.message.includes("No object generated") || lastError.name.includes("AI_");
|
|
3363
|
+
if (!isRetryable) {
|
|
3364
|
+
throw lastError;
|
|
2143
3365
|
}
|
|
2144
3366
|
}
|
|
2145
|
-
throw lastError;
|
|
2146
3367
|
}
|
|
2147
|
-
|
|
3368
|
+
throw lastError;
|
|
3369
|
+
}
|
|
2148
3370
|
export {
|
|
2149
3371
|
ALL_STYLES,
|
|
2150
3372
|
BaseContextualExtractor,
|
|
@@ -2156,15 +3378,15 @@ export {
|
|
|
2156
3378
|
LastQueryExtractor,
|
|
2157
3379
|
MessageExtractor,
|
|
2158
3380
|
PairProducer,
|
|
2159
|
-
PersonaGenerator,
|
|
2160
3381
|
SchemaSynthesizer,
|
|
2161
3382
|
SegmentedContextExtractor,
|
|
2162
3383
|
SqlExtractor,
|
|
2163
|
-
TeachingsGenerator,
|
|
2164
3384
|
ValidatedProducer,
|
|
2165
3385
|
WindowedContextExtractor,
|
|
2166
3386
|
contextResolverAgent,
|
|
2167
3387
|
formatConversation,
|
|
3388
|
+
generatePersonas,
|
|
3389
|
+
generateTeachings,
|
|
2168
3390
|
getMessageText,
|
|
2169
3391
|
styleInstructions,
|
|
2170
3392
|
toPairs
|