llmist 15.10.0 → 15.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1034 -828
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +191 -1
- package/dist/index.d.ts +191 -1
- package/dist/index.js +1032 -828
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -1785,9 +1785,25 @@ function resolveRetryConfig(config) {
|
|
|
1785
1785
|
maxRetryAfterMs: config.maxRetryAfterMs ?? DEFAULT_RETRY_CONFIG.maxRetryAfterMs
|
|
1786
1786
|
};
|
|
1787
1787
|
}
|
|
1788
|
+
function getErrorStatusCode(error) {
|
|
1789
|
+
const errAny = error;
|
|
1790
|
+
if (typeof errAny.status === "number") return errAny.status;
|
|
1791
|
+
if (typeof errAny.code === "number") return errAny.code;
|
|
1792
|
+
if (typeof errAny.statusCode === "number") return errAny.statusCode;
|
|
1793
|
+
if (typeof errAny.code === "string" && /^\d{3}$/.test(errAny.code)) {
|
|
1794
|
+
return parseInt(errAny.code, 10);
|
|
1795
|
+
}
|
|
1796
|
+
return void 0;
|
|
1797
|
+
}
|
|
1788
1798
|
function isRetryableError(error) {
|
|
1789
1799
|
const message = error.message.toLowerCase();
|
|
1790
1800
|
const name = error.name;
|
|
1801
|
+
const statusCode = getErrorStatusCode(error);
|
|
1802
|
+
if (statusCode !== void 0) {
|
|
1803
|
+
if (statusCode === 429) return true;
|
|
1804
|
+
if (statusCode >= 500 && statusCode < 600) return true;
|
|
1805
|
+
if (statusCode >= 400 && statusCode < 500) return false;
|
|
1806
|
+
}
|
|
1791
1807
|
if (message.includes("429") || message.includes("rate limit") || message.includes("rate_limit")) {
|
|
1792
1808
|
return true;
|
|
1793
1809
|
}
|
|
@@ -3958,66 +3974,1015 @@ var init_registry = __esm({
|
|
|
3958
3974
|
registry.register(name, instance);
|
|
3959
3975
|
}
|
|
3960
3976
|
}
|
|
3961
|
-
return registry;
|
|
3977
|
+
return registry;
|
|
3978
|
+
}
|
|
3979
|
+
/**
|
|
3980
|
+
* Registers multiple gadgets at once from an array.
|
|
3981
|
+
*
|
|
3982
|
+
* @param gadgets - Array of gadget instances or classes
|
|
3983
|
+
* @returns This registry for chaining
|
|
3984
|
+
*
|
|
3985
|
+
* @example
|
|
3986
|
+
* ```typescript
|
|
3987
|
+
* registry.registerMany([Calculator, Weather, Email]);
|
|
3988
|
+
* registry.registerMany([new Calculator(), new Weather()]);
|
|
3989
|
+
* ```
|
|
3990
|
+
*/
|
|
3991
|
+
registerMany(gadgets) {
|
|
3992
|
+
for (const gadget of gadgets) {
|
|
3993
|
+
const instance = typeof gadget === "function" ? new gadget() : gadget;
|
|
3994
|
+
this.registerByClass(instance);
|
|
3995
|
+
}
|
|
3996
|
+
return this;
|
|
3997
|
+
}
|
|
3998
|
+
// Register a gadget by name
|
|
3999
|
+
register(name, gadget) {
|
|
4000
|
+
const normalizedName = name.toLowerCase();
|
|
4001
|
+
if (this.gadgets.has(normalizedName)) {
|
|
4002
|
+
throw new Error(`Gadget '${name}' is already registered`);
|
|
4003
|
+
}
|
|
4004
|
+
if (gadget.parameterSchema) {
|
|
4005
|
+
validateGadgetSchema(gadget.parameterSchema, name);
|
|
4006
|
+
}
|
|
4007
|
+
this.gadgets.set(normalizedName, gadget);
|
|
4008
|
+
}
|
|
4009
|
+
// Register a gadget using its name property or class name
|
|
4010
|
+
registerByClass(gadget) {
|
|
4011
|
+
const name = gadget.name ?? gadget.constructor.name;
|
|
4012
|
+
this.register(name, gadget);
|
|
4013
|
+
}
|
|
4014
|
+
// Get gadget by name (case-insensitive)
|
|
4015
|
+
get(name) {
|
|
4016
|
+
return this.gadgets.get(name.toLowerCase());
|
|
4017
|
+
}
|
|
4018
|
+
// Check if gadget exists (case-insensitive)
|
|
4019
|
+
has(name) {
|
|
4020
|
+
return this.gadgets.has(name.toLowerCase());
|
|
4021
|
+
}
|
|
4022
|
+
// Get all registered gadget names
|
|
4023
|
+
getNames() {
|
|
4024
|
+
return Array.from(this.gadgets.keys());
|
|
4025
|
+
}
|
|
4026
|
+
// Get all gadgets for instruction generation
|
|
4027
|
+
getAll() {
|
|
4028
|
+
return Array.from(this.gadgets.values());
|
|
4029
|
+
}
|
|
4030
|
+
// Unregister gadget (useful for testing, case-insensitive)
|
|
4031
|
+
unregister(name) {
|
|
4032
|
+
return this.gadgets.delete(name.toLowerCase());
|
|
4033
|
+
}
|
|
4034
|
+
// Clear all gadgets (useful for testing)
|
|
4035
|
+
clear() {
|
|
4036
|
+
this.gadgets.clear();
|
|
4037
|
+
}
|
|
4038
|
+
};
|
|
4039
|
+
}
|
|
4040
|
+
});
|
|
4041
|
+
|
|
4042
|
+
// src/agent/file-logging.ts
|
|
4043
|
+
function formatLlmRequest(messages) {
|
|
4044
|
+
const lines = [];
|
|
4045
|
+
for (const msg of messages) {
|
|
4046
|
+
lines.push(`=== ${msg.role.toUpperCase()} ===`);
|
|
4047
|
+
lines.push(msg.content ? extractMessageText(msg.content) : "");
|
|
4048
|
+
lines.push("");
|
|
4049
|
+
}
|
|
4050
|
+
return lines.join("\n");
|
|
4051
|
+
}
|
|
4052
|
+
function formatCallNumber(n, padding = 4) {
|
|
4053
|
+
return n.toString().padStart(padding, "0");
|
|
4054
|
+
}
|
|
4055
|
+
async function writeLogFile(dir, filename, content) {
|
|
4056
|
+
await (0, import_promises2.mkdir)(dir, { recursive: true });
|
|
4057
|
+
await (0, import_promises2.writeFile)((0, import_node_path3.join)(dir, filename), content, "utf-8");
|
|
4058
|
+
}
|
|
4059
|
+
function createFileLoggingHooks(options) {
|
|
4060
|
+
const {
|
|
4061
|
+
directory,
|
|
4062
|
+
startingCounter = 1,
|
|
4063
|
+
counterPadding = 4,
|
|
4064
|
+
skipSubagents = true,
|
|
4065
|
+
formatRequest = formatLlmRequest,
|
|
4066
|
+
onFileWritten
|
|
4067
|
+
} = options;
|
|
4068
|
+
let callCounter = startingCounter - 1;
|
|
4069
|
+
return {
|
|
4070
|
+
observers: {
|
|
4071
|
+
/**
|
|
4072
|
+
* Write request file when LLM call is ready (messages are finalized).
|
|
4073
|
+
*/
|
|
4074
|
+
onLLMCallReady: async (context) => {
|
|
4075
|
+
if (skipSubagents && context.subagentContext) {
|
|
4076
|
+
return;
|
|
4077
|
+
}
|
|
4078
|
+
callCounter++;
|
|
4079
|
+
const filename = `${formatCallNumber(callCounter, counterPadding)}.request`;
|
|
4080
|
+
const content = formatRequest(context.options.messages);
|
|
4081
|
+
try {
|
|
4082
|
+
await writeLogFile(directory, filename, content);
|
|
4083
|
+
if (onFileWritten) {
|
|
4084
|
+
onFileWritten({
|
|
4085
|
+
filePath: (0, import_node_path3.join)(directory, filename),
|
|
4086
|
+
type: "request",
|
|
4087
|
+
callNumber: callCounter,
|
|
4088
|
+
contentLength: content.length
|
|
4089
|
+
});
|
|
4090
|
+
}
|
|
4091
|
+
} catch (error) {
|
|
4092
|
+
console.warn(`[file-logging] Failed to write ${filename}:`, error);
|
|
4093
|
+
}
|
|
4094
|
+
},
|
|
4095
|
+
/**
|
|
4096
|
+
* Write response file when LLM call completes.
|
|
4097
|
+
*/
|
|
4098
|
+
onLLMCallComplete: async (context) => {
|
|
4099
|
+
if (skipSubagents && context.subagentContext) {
|
|
4100
|
+
return;
|
|
4101
|
+
}
|
|
4102
|
+
const filename = `${formatCallNumber(callCounter, counterPadding)}.response`;
|
|
4103
|
+
const content = context.rawResponse;
|
|
4104
|
+
try {
|
|
4105
|
+
await writeLogFile(directory, filename, content);
|
|
4106
|
+
if (onFileWritten) {
|
|
4107
|
+
onFileWritten({
|
|
4108
|
+
filePath: (0, import_node_path3.join)(directory, filename),
|
|
4109
|
+
type: "response",
|
|
4110
|
+
callNumber: callCounter,
|
|
4111
|
+
contentLength: content.length
|
|
4112
|
+
});
|
|
4113
|
+
}
|
|
4114
|
+
} catch (error) {
|
|
4115
|
+
console.warn(`[file-logging] Failed to write ${filename}:`, error);
|
|
4116
|
+
}
|
|
4117
|
+
}
|
|
4118
|
+
}
|
|
4119
|
+
};
|
|
4120
|
+
}
|
|
4121
|
+
function getEnvFileLoggingHooks() {
|
|
4122
|
+
const directory = process.env[ENV_LOG_RAW_DIRECTORY]?.trim();
|
|
4123
|
+
if (!directory) {
|
|
4124
|
+
return void 0;
|
|
4125
|
+
}
|
|
4126
|
+
return createFileLoggingHooks({ directory });
|
|
4127
|
+
}
|
|
4128
|
+
var import_promises2, import_node_path3, ENV_LOG_RAW_DIRECTORY;
|
|
4129
|
+
var init_file_logging = __esm({
|
|
4130
|
+
"src/agent/file-logging.ts"() {
|
|
4131
|
+
"use strict";
|
|
4132
|
+
import_promises2 = require("fs/promises");
|
|
4133
|
+
import_node_path3 = require("path");
|
|
4134
|
+
init_messages();
|
|
4135
|
+
ENV_LOG_RAW_DIRECTORY = "LLMIST_LOG_RAW_DIRECTORY";
|
|
4136
|
+
}
|
|
4137
|
+
});
|
|
4138
|
+
|
|
4139
|
+
// src/agent/hook-presets.ts
|
|
4140
|
+
var HookPresets;
|
|
4141
|
+
var init_hook_presets = __esm({
|
|
4142
|
+
"src/agent/hook-presets.ts"() {
|
|
4143
|
+
"use strict";
|
|
4144
|
+
init_file_logging();
|
|
4145
|
+
HookPresets = class _HookPresets {
|
|
4146
|
+
/**
|
|
4147
|
+
* Logs LLM calls and gadget execution to console with optional verbosity.
|
|
4148
|
+
*
|
|
4149
|
+
* **Output (basic mode):**
|
|
4150
|
+
* - LLM call start/complete events with iteration numbers
|
|
4151
|
+
* - Gadget execution start/complete with gadget names
|
|
4152
|
+
* - Token counts when available
|
|
4153
|
+
*
|
|
4154
|
+
* **Output (verbose mode):**
|
|
4155
|
+
* - All basic mode output
|
|
4156
|
+
* - Full gadget parameters (formatted JSON)
|
|
4157
|
+
* - Full gadget results
|
|
4158
|
+
* - Complete LLM response text
|
|
4159
|
+
*
|
|
4160
|
+
* **Use cases:**
|
|
4161
|
+
* - Basic development debugging and execution flow visibility
|
|
4162
|
+
* - Understanding agent decision-making and tool usage
|
|
4163
|
+
* - Troubleshooting gadget invocations
|
|
4164
|
+
*
|
|
4165
|
+
* **Performance:** Minimal overhead. Console writes are synchronous but fast.
|
|
4166
|
+
*
|
|
4167
|
+
* @param options - Logging options
|
|
4168
|
+
* @param options.verbose - Include full parameters and results. Default: false
|
|
4169
|
+
* @returns Hook configuration that can be passed to .withHooks()
|
|
4170
|
+
*
|
|
4171
|
+
* @example
|
|
4172
|
+
* ```typescript
|
|
4173
|
+
* // Basic logging
|
|
4174
|
+
* await LLMist.createAgent()
|
|
4175
|
+
* .withHooks(HookPresets.logging())
|
|
4176
|
+
* .ask("Calculate 15 * 23");
|
|
4177
|
+
* // Output: [LLM] Starting call (iteration 0)
|
|
4178
|
+
* // [GADGET] Executing Calculator
|
|
4179
|
+
* // [GADGET] Completed Calculator
|
|
4180
|
+
* // [LLM] Completed (tokens: 245)
|
|
4181
|
+
* ```
|
|
4182
|
+
*
|
|
4183
|
+
* @example
|
|
4184
|
+
* ```typescript
|
|
4185
|
+
* // Verbose logging with full details
|
|
4186
|
+
* await LLMist.createAgent()
|
|
4187
|
+
* .withHooks(HookPresets.logging({ verbose: true }))
|
|
4188
|
+
* .ask("Calculate 15 * 23");
|
|
4189
|
+
* // Output includes: parameters, results, and full responses
|
|
4190
|
+
* ```
|
|
4191
|
+
*
|
|
4192
|
+
* @example
|
|
4193
|
+
* ```typescript
|
|
4194
|
+
* // Environment-based verbosity
|
|
4195
|
+
* const isDev = process.env.NODE_ENV === 'development';
|
|
4196
|
+
* .withHooks(HookPresets.logging({ verbose: isDev }))
|
|
4197
|
+
* ```
|
|
4198
|
+
*
|
|
4199
|
+
* @see {@link https://github.com/zbigniewsobiecki/llmist/blob/main/docs/HOOKS.md#hookpresetsloggingoptions | Full documentation}
|
|
4200
|
+
*/
|
|
4201
|
+
static logging(options = {}) {
|
|
4202
|
+
return {
|
|
4203
|
+
observers: {
|
|
4204
|
+
onLLMCallStart: async (ctx) => {
|
|
4205
|
+
console.log(`[LLM] Starting call (iteration ${ctx.iteration})`);
|
|
4206
|
+
},
|
|
4207
|
+
onLLMCallComplete: async (ctx) => {
|
|
4208
|
+
const tokens = ctx.usage?.totalTokens ?? "unknown";
|
|
4209
|
+
console.log(`[LLM] Completed (tokens: ${tokens})`);
|
|
4210
|
+
if (options.verbose && ctx.finalMessage) {
|
|
4211
|
+
console.log(`[LLM] Response: ${ctx.finalMessage}`);
|
|
4212
|
+
}
|
|
4213
|
+
},
|
|
4214
|
+
onGadgetExecutionStart: async (ctx) => {
|
|
4215
|
+
console.log(`[GADGET] Executing ${ctx.gadgetName}`);
|
|
4216
|
+
if (options.verbose) {
|
|
4217
|
+
console.log(`[GADGET] Parameters:`, JSON.stringify(ctx.parameters, null, 2));
|
|
4218
|
+
}
|
|
4219
|
+
},
|
|
4220
|
+
onGadgetExecutionComplete: async (ctx) => {
|
|
4221
|
+
console.log(`[GADGET] Completed ${ctx.gadgetName}`);
|
|
4222
|
+
if (options.verbose) {
|
|
4223
|
+
const display = ctx.error ?? ctx.finalResult ?? "(no result)";
|
|
4224
|
+
console.log(`[GADGET] Result: ${display}`);
|
|
4225
|
+
}
|
|
4226
|
+
}
|
|
4227
|
+
}
|
|
4228
|
+
};
|
|
4229
|
+
}
|
|
4230
|
+
/**
|
|
4231
|
+
* Measures and logs execution time for LLM calls and gadgets.
|
|
4232
|
+
*
|
|
4233
|
+
* **Output:**
|
|
4234
|
+
* - Duration in milliseconds with ⏱️ emoji for each operation
|
|
4235
|
+
* - Separate timing for each LLM iteration
|
|
4236
|
+
* - Separate timing for each gadget execution
|
|
4237
|
+
*
|
|
4238
|
+
* **Use cases:**
|
|
4239
|
+
* - Performance profiling and optimization
|
|
4240
|
+
* - Identifying slow operations (LLM calls vs gadget execution)
|
|
4241
|
+
* - Monitoring response times in production
|
|
4242
|
+
* - Capacity planning and SLA tracking
|
|
4243
|
+
*
|
|
4244
|
+
* **Performance:** Negligible overhead. Uses Date.now() for timing measurements.
|
|
4245
|
+
*
|
|
4246
|
+
* @returns Hook configuration that can be passed to .withHooks()
|
|
4247
|
+
*
|
|
4248
|
+
* @example
|
|
4249
|
+
* ```typescript
|
|
4250
|
+
* // Basic timing
|
|
4251
|
+
* await LLMist.createAgent()
|
|
4252
|
+
* .withHooks(HookPresets.timing())
|
|
4253
|
+
* .withGadgets(Weather, Database)
|
|
4254
|
+
* .ask("What's the weather in NYC?");
|
|
4255
|
+
* // Output: ⏱️ LLM call took 1234ms
|
|
4256
|
+
* // ⏱️ Gadget Weather took 567ms
|
|
4257
|
+
* // ⏱️ LLM call took 890ms
|
|
4258
|
+
* ```
|
|
4259
|
+
*
|
|
4260
|
+
* @example
|
|
4261
|
+
* ```typescript
|
|
4262
|
+
* // Combined with logging for full context
|
|
4263
|
+
* .withHooks(HookPresets.merge(
|
|
4264
|
+
* HookPresets.logging(),
|
|
4265
|
+
* HookPresets.timing()
|
|
4266
|
+
* ))
|
|
4267
|
+
* ```
|
|
4268
|
+
*
|
|
4269
|
+
* @example
|
|
4270
|
+
* ```typescript
|
|
4271
|
+
* // Correlate performance with cost
|
|
4272
|
+
* .withHooks(HookPresets.merge(
|
|
4273
|
+
* HookPresets.timing(),
|
|
4274
|
+
* HookPresets.tokenTracking()
|
|
4275
|
+
* ))
|
|
4276
|
+
* ```
|
|
4277
|
+
*
|
|
4278
|
+
* @see {@link https://github.com/zbigniewsobiecki/llmist/blob/main/docs/HOOKS.md#hookpresetstiming | Full documentation}
|
|
4279
|
+
*/
|
|
4280
|
+
static timing() {
|
|
4281
|
+
const timings = /* @__PURE__ */ new Map();
|
|
4282
|
+
return {
|
|
4283
|
+
observers: {
|
|
4284
|
+
onLLMCallStart: async (ctx) => {
|
|
4285
|
+
timings.set(`llm-${ctx.iteration}`, Date.now());
|
|
4286
|
+
},
|
|
4287
|
+
onLLMCallComplete: async (ctx) => {
|
|
4288
|
+
const start = timings.get(`llm-${ctx.iteration}`);
|
|
4289
|
+
if (start) {
|
|
4290
|
+
const duration = Date.now() - start;
|
|
4291
|
+
console.log(`\u23F1\uFE0F LLM call took ${duration}ms`);
|
|
4292
|
+
timings.delete(`llm-${ctx.iteration}`);
|
|
4293
|
+
}
|
|
4294
|
+
},
|
|
4295
|
+
onGadgetExecutionStart: async (ctx) => {
|
|
4296
|
+
const key = `gadget-${ctx.gadgetName}-${Date.now()}`;
|
|
4297
|
+
timings.set(key, Date.now());
|
|
4298
|
+
ctx._timingKey = key;
|
|
4299
|
+
},
|
|
4300
|
+
onGadgetExecutionComplete: async (ctx) => {
|
|
4301
|
+
const key = ctx._timingKey;
|
|
4302
|
+
if (key) {
|
|
4303
|
+
const start = timings.get(key);
|
|
4304
|
+
if (start) {
|
|
4305
|
+
const duration = Date.now() - start;
|
|
4306
|
+
console.log(`\u23F1\uFE0F Gadget ${ctx.gadgetName} took ${duration}ms`);
|
|
4307
|
+
timings.delete(key);
|
|
4308
|
+
}
|
|
4309
|
+
}
|
|
4310
|
+
}
|
|
4311
|
+
}
|
|
4312
|
+
};
|
|
4313
|
+
}
|
|
4314
|
+
/**
|
|
4315
|
+
* Tracks cumulative token usage across all LLM calls.
|
|
4316
|
+
*
|
|
4317
|
+
* **Output:**
|
|
4318
|
+
* - Per-call token count with 📊 emoji
|
|
4319
|
+
* - Cumulative total across all calls
|
|
4320
|
+
* - Call count for average calculations
|
|
4321
|
+
*
|
|
4322
|
+
* **Use cases:**
|
|
4323
|
+
* - Cost monitoring and budget tracking
|
|
4324
|
+
* - Optimizing prompts to reduce token usage
|
|
4325
|
+
* - Comparing token efficiency across different approaches
|
|
4326
|
+
* - Real-time cost estimation
|
|
4327
|
+
*
|
|
4328
|
+
* **Performance:** Minimal overhead. Simple counter increments.
|
|
4329
|
+
*
|
|
4330
|
+
* **Note:** Token counts depend on the provider's response. Some providers
|
|
4331
|
+
* may not include usage data, in which case counts won't be logged.
|
|
4332
|
+
*
|
|
4333
|
+
* @returns Hook configuration that can be passed to .withHooks()
|
|
4334
|
+
*
|
|
4335
|
+
* @example
|
|
4336
|
+
* ```typescript
|
|
4337
|
+
* // Basic token tracking
|
|
4338
|
+
* await LLMist.createAgent()
|
|
4339
|
+
* .withHooks(HookPresets.tokenTracking())
|
|
4340
|
+
* .ask("Summarize this document...");
|
|
4341
|
+
* // Output: 📊 Tokens this call: 1,234
|
|
4342
|
+
* // 📊 Total tokens: 1,234 (across 1 calls)
|
|
4343
|
+
* // 📊 Tokens this call: 567
|
|
4344
|
+
* // 📊 Total tokens: 1,801 (across 2 calls)
|
|
4345
|
+
* ```
|
|
4346
|
+
*
|
|
4347
|
+
* @example
|
|
4348
|
+
* ```typescript
|
|
4349
|
+
* // Cost calculation with custom hook
|
|
4350
|
+
* let totalTokens = 0;
|
|
4351
|
+
* .withHooks(HookPresets.merge(
|
|
4352
|
+
* HookPresets.tokenTracking(),
|
|
4353
|
+
* {
|
|
4354
|
+
* observers: {
|
|
4355
|
+
* onLLMCallComplete: async (ctx) => {
|
|
4356
|
+
* totalTokens += ctx.usage?.totalTokens ?? 0;
|
|
4357
|
+
* const cost = (totalTokens / 1_000_000) * 3.0; // $3 per 1M tokens
|
|
4358
|
+
* console.log(`💰 Estimated cost: $${cost.toFixed(4)}`);
|
|
4359
|
+
* },
|
|
4360
|
+
* },
|
|
4361
|
+
* }
|
|
4362
|
+
* ))
|
|
4363
|
+
* ```
|
|
4364
|
+
*
|
|
4365
|
+
* @see {@link https://github.com/zbigniewsobiecki/llmist/blob/main/docs/HOOKS.md#hookpresetstokentracking | Full documentation}
|
|
4366
|
+
*/
|
|
4367
|
+
static tokenTracking() {
|
|
4368
|
+
let totalTokens = 0;
|
|
4369
|
+
let totalCalls = 0;
|
|
4370
|
+
return {
|
|
4371
|
+
observers: {
|
|
4372
|
+
onLLMCallComplete: async (ctx) => {
|
|
4373
|
+
totalCalls++;
|
|
4374
|
+
if (ctx.usage?.totalTokens) {
|
|
4375
|
+
totalTokens += ctx.usage.totalTokens;
|
|
4376
|
+
console.log(`\u{1F4CA} Tokens this call: ${ctx.usage.totalTokens}`);
|
|
4377
|
+
console.log(`\u{1F4CA} Total tokens: ${totalTokens} (across ${totalCalls} calls)`);
|
|
4378
|
+
}
|
|
4379
|
+
}
|
|
4380
|
+
}
|
|
4381
|
+
};
|
|
4382
|
+
}
|
|
4383
|
+
/**
|
|
4384
|
+
* Tracks comprehensive progress metrics including iterations, tokens, cost, and timing.
|
|
4385
|
+
*
|
|
4386
|
+
* **This preset showcases llmist's core capabilities by demonstrating:**
|
|
4387
|
+
* - Observer pattern for non-intrusive monitoring
|
|
4388
|
+
* - Integration with ModelRegistry for cost estimation
|
|
4389
|
+
* - Callback-based architecture for flexible UI updates
|
|
4390
|
+
* - Provider-agnostic token and cost tracking
|
|
4391
|
+
*
|
|
4392
|
+
* Unlike `tokenTracking()` which only logs to console, this preset provides
|
|
4393
|
+
* structured data through callbacks, making it perfect for building custom UIs,
|
|
4394
|
+
* dashboards, or progress indicators (like the llmist CLI).
|
|
4395
|
+
*
|
|
4396
|
+
* **Output (when logProgress: true):**
|
|
4397
|
+
* - Iteration number and call count
|
|
4398
|
+
* - Cumulative token usage (input + output)
|
|
4399
|
+
* - Cumulative cost in USD (requires modelRegistry)
|
|
4400
|
+
* - Elapsed time in seconds
|
|
4401
|
+
*
|
|
4402
|
+
* **Use cases:**
|
|
4403
|
+
* - Building CLI progress indicators with live updates
|
|
4404
|
+
* - Creating web dashboards with real-time metrics
|
|
4405
|
+
* - Budget monitoring and cost alerts
|
|
4406
|
+
* - Performance tracking and optimization
|
|
4407
|
+
* - Custom logging to external systems (Datadog, CloudWatch, etc.)
|
|
4408
|
+
*
|
|
4409
|
+
* **Performance:** Minimal overhead. Uses Date.now() for timing and optional
|
|
4410
|
+
* ModelRegistry.estimateCost() which is O(1) lookup. Callback invocation is
|
|
4411
|
+
* synchronous and fast.
|
|
4412
|
+
*
|
|
4413
|
+
* @param options - Progress tracking options
|
|
4414
|
+
* @param options.modelRegistry - ModelRegistry for cost estimation (optional)
|
|
4415
|
+
* @param options.onProgress - Callback invoked after each LLM call (optional)
|
|
4416
|
+
* @param options.logProgress - Log progress to console (default: false)
|
|
4417
|
+
* @returns Hook configuration with progress tracking observers
|
|
4418
|
+
*
|
|
4419
|
+
* @example
|
|
4420
|
+
* ```typescript
|
|
4421
|
+
* // Basic usage with callback (RECOMMENDED - used by llmist CLI)
|
|
4422
|
+
* import { LLMist, HookPresets } from 'llmist';
|
|
4423
|
+
*
|
|
4424
|
+
* const client = LLMist.create();
|
|
4425
|
+
*
|
|
4426
|
+
* await client.agent()
|
|
4427
|
+
* .withHooks(HookPresets.progressTracking({
|
|
4428
|
+
* modelRegistry: client.modelRegistry,
|
|
4429
|
+
* onProgress: (stats) => {
|
|
4430
|
+
* // Update your UI with stats
|
|
4431
|
+
* console.log(`#${stats.currentIteration} | ${stats.totalTokens} tokens | $${stats.totalCost.toFixed(4)}`);
|
|
4432
|
+
* }
|
|
4433
|
+
* }))
|
|
4434
|
+
* .withGadgets(Calculator)
|
|
4435
|
+
* .ask("Calculate 15 * 23");
|
|
4436
|
+
* // Output: #1 | 245 tokens | $0.0012
|
|
4437
|
+
* ```
|
|
4438
|
+
*
|
|
4439
|
+
* @example
|
|
4440
|
+
* ```typescript
|
|
4441
|
+
* // Console logging mode (quick debugging)
|
|
4442
|
+
* await client.agent()
|
|
4443
|
+
* .withHooks(HookPresets.progressTracking({
|
|
4444
|
+
* modelRegistry: client.modelRegistry,
|
|
4445
|
+
* logProgress: true // Simple console output
|
|
4446
|
+
* }))
|
|
4447
|
+
* .ask("Your prompt");
|
|
4448
|
+
* // Output: 📊 Progress: Iteration #1 | 245 tokens | $0.0012 | 1.2s
|
|
4449
|
+
* ```
|
|
4450
|
+
*
|
|
4451
|
+
* @example
|
|
4452
|
+
* ```typescript
|
|
4453
|
+
* // Budget monitoring with alerts
|
|
4454
|
+
* const BUDGET_USD = 0.10;
|
|
4455
|
+
*
|
|
4456
|
+
* await client.agent()
|
|
4457
|
+
* .withHooks(HookPresets.progressTracking({
|
|
4458
|
+
* modelRegistry: client.modelRegistry,
|
|
4459
|
+
* onProgress: (stats) => {
|
|
4460
|
+
* if (stats.totalCost > BUDGET_USD) {
|
|
4461
|
+
* throw new Error(`Budget exceeded: $${stats.totalCost.toFixed(4)}`);
|
|
4462
|
+
* }
|
|
4463
|
+
* }
|
|
4464
|
+
* }))
|
|
4465
|
+
* .ask("Long running task...");
|
|
4466
|
+
* ```
|
|
4467
|
+
*
|
|
4468
|
+
* @example
|
|
4469
|
+
* ```typescript
|
|
4470
|
+
* // Web dashboard integration
|
|
4471
|
+
* let progressBar: HTMLElement;
|
|
4472
|
+
*
|
|
4473
|
+
* await client.agent()
|
|
4474
|
+
* .withHooks(HookPresets.progressTracking({
|
|
4475
|
+
* modelRegistry: client.modelRegistry,
|
|
4476
|
+
* onProgress: (stats) => {
|
|
4477
|
+
* // Update web UI in real-time
|
|
4478
|
+
* progressBar.textContent = `Iteration ${stats.currentIteration}`;
|
|
4479
|
+
* progressBar.dataset.cost = stats.totalCost.toFixed(4);
|
|
4480
|
+
* progressBar.dataset.tokens = stats.totalTokens.toString();
|
|
4481
|
+
* }
|
|
4482
|
+
* }))
|
|
4483
|
+
* .ask("Your prompt");
|
|
4484
|
+
* ```
|
|
4485
|
+
*
|
|
4486
|
+
* @example
|
|
4487
|
+
* ```typescript
|
|
4488
|
+
* // External logging (Datadog, CloudWatch, etc.)
|
|
4489
|
+
* await client.agent()
|
|
4490
|
+
* .withHooks(HookPresets.progressTracking({
|
|
4491
|
+
* modelRegistry: client.modelRegistry,
|
|
4492
|
+
* onProgress: async (stats) => {
|
|
4493
|
+
* await metrics.gauge('llm.iteration', stats.currentIteration);
|
|
4494
|
+
* await metrics.gauge('llm.cost', stats.totalCost);
|
|
4495
|
+
* await metrics.gauge('llm.tokens', stats.totalTokens);
|
|
4496
|
+
* }
|
|
4497
|
+
* }))
|
|
4498
|
+
* .ask("Your prompt");
|
|
4499
|
+
* ```
|
|
4500
|
+
*
|
|
4501
|
+
* @see {@link https://github.com/zbigniewsobiecki/llmist/blob/main/docs/HOOKS.md#hookpresetsprogresstrackingoptions | Full documentation}
|
|
4502
|
+
* @see {@link ProgressTrackingOptions} for detailed options
|
|
4503
|
+
* @see {@link ProgressStats} for the callback data structure
|
|
4504
|
+
*/
|
|
4505
|
+
static progressTracking(options) {
|
|
4506
|
+
const { modelRegistry, onProgress, logProgress = false } = options ?? {};
|
|
4507
|
+
let totalCalls = 0;
|
|
4508
|
+
let currentIteration = 0;
|
|
4509
|
+
let totalInputTokens = 0;
|
|
4510
|
+
let totalOutputTokens = 0;
|
|
4511
|
+
let totalCost = 0;
|
|
4512
|
+
let totalGadgetCost = 0;
|
|
4513
|
+
const startTime = Date.now();
|
|
4514
|
+
return {
|
|
4515
|
+
observers: {
|
|
4516
|
+
// Track iteration on each LLM call start
|
|
4517
|
+
onLLMCallStart: async (ctx) => {
|
|
4518
|
+
currentIteration++;
|
|
4519
|
+
},
|
|
4520
|
+
// Accumulate metrics and report progress on each LLM call completion
|
|
4521
|
+
onLLMCallComplete: async (ctx) => {
|
|
4522
|
+
totalCalls++;
|
|
4523
|
+
if (ctx.usage) {
|
|
4524
|
+
totalInputTokens += ctx.usage.inputTokens;
|
|
4525
|
+
totalOutputTokens += ctx.usage.outputTokens;
|
|
4526
|
+
if (modelRegistry) {
|
|
4527
|
+
try {
|
|
4528
|
+
const modelName = ctx.options.model.includes(":") ? ctx.options.model.split(":")[1] : ctx.options.model;
|
|
4529
|
+
const costEstimate = modelRegistry.estimateCost(
|
|
4530
|
+
modelName,
|
|
4531
|
+
ctx.usage.inputTokens,
|
|
4532
|
+
ctx.usage.outputTokens
|
|
4533
|
+
);
|
|
4534
|
+
if (costEstimate) {
|
|
4535
|
+
totalCost += costEstimate.totalCost;
|
|
4536
|
+
}
|
|
4537
|
+
} catch (error) {
|
|
4538
|
+
if (logProgress) {
|
|
4539
|
+
console.warn(`\u26A0\uFE0F Cost estimation failed:`, error);
|
|
4540
|
+
}
|
|
4541
|
+
}
|
|
4542
|
+
}
|
|
4543
|
+
}
|
|
4544
|
+
const stats = {
|
|
4545
|
+
currentIteration,
|
|
4546
|
+
totalCalls,
|
|
4547
|
+
totalInputTokens,
|
|
4548
|
+
totalOutputTokens,
|
|
4549
|
+
totalTokens: totalInputTokens + totalOutputTokens,
|
|
4550
|
+
totalCost: totalCost + totalGadgetCost,
|
|
4551
|
+
elapsedSeconds: Number(((Date.now() - startTime) / 1e3).toFixed(1))
|
|
4552
|
+
};
|
|
4553
|
+
if (onProgress) {
|
|
4554
|
+
onProgress(stats);
|
|
4555
|
+
}
|
|
4556
|
+
if (logProgress) {
|
|
4557
|
+
const formattedTokens = stats.totalTokens >= 1e3 ? `${(stats.totalTokens / 1e3).toFixed(1)}k` : `${stats.totalTokens}`;
|
|
4558
|
+
const formattedCost = stats.totalCost > 0 ? `$${stats.totalCost.toFixed(4)}` : "$0";
|
|
4559
|
+
console.log(
|
|
4560
|
+
`\u{1F4CA} Progress: Iteration #${stats.currentIteration} | ${formattedTokens} tokens | ${formattedCost} | ${stats.elapsedSeconds}s`
|
|
4561
|
+
);
|
|
4562
|
+
}
|
|
4563
|
+
},
|
|
4564
|
+
// Track gadget execution costs
|
|
4565
|
+
onGadgetExecutionComplete: async (ctx) => {
|
|
4566
|
+
if (ctx.cost && ctx.cost > 0) {
|
|
4567
|
+
totalGadgetCost += ctx.cost;
|
|
4568
|
+
}
|
|
4569
|
+
}
|
|
4570
|
+
}
|
|
4571
|
+
};
|
|
4572
|
+
}
|
|
4573
|
+
/**
|
|
4574
|
+
* Logs detailed error information for debugging and troubleshooting.
|
|
4575
|
+
*
|
|
4576
|
+
* **Output:**
|
|
4577
|
+
* - LLM errors with ❌ emoji, including model and recovery status
|
|
4578
|
+
* - Gadget errors with full context (parameters, error message)
|
|
4579
|
+
* - Separate logging for LLM and gadget failures
|
|
4580
|
+
*
|
|
4581
|
+
* **Use cases:**
|
|
4582
|
+
* - Troubleshooting production issues
|
|
4583
|
+
* - Understanding error patterns and frequency
|
|
4584
|
+
* - Debugging error recovery behavior
|
|
4585
|
+
* - Collecting error metrics for monitoring
|
|
4586
|
+
*
|
|
4587
|
+
* **Performance:** Minimal overhead. Only logs when errors occur.
|
|
4588
|
+
*
|
|
4589
|
+
* @returns Hook configuration that can be passed to .withHooks()
|
|
4590
|
+
*
|
|
4591
|
+
* @example
|
|
4592
|
+
* ```typescript
|
|
4593
|
+
* // Basic error logging
|
|
4594
|
+
* await LLMist.createAgent()
|
|
4595
|
+
* .withHooks(HookPresets.errorLogging())
|
|
4596
|
+
* .withGadgets(Database)
|
|
4597
|
+
* .ask("Fetch user data");
|
|
4598
|
+
* // Output (on LLM error): ❌ LLM Error (iteration 1): Rate limit exceeded
|
|
4599
|
+
* // Model: gpt-5-nano
|
|
4600
|
+
* // Recovered: true
|
|
4601
|
+
* // Output (on gadget error): ❌ Gadget Error: Database
|
|
4602
|
+
* // Error: Connection timeout
|
|
4603
|
+
* // Parameters: {...}
|
|
4604
|
+
* ```
|
|
4605
|
+
*
|
|
4606
|
+
* @example
|
|
4607
|
+
* ```typescript
|
|
4608
|
+
* // Combine with monitoring for full context
|
|
4609
|
+
* .withHooks(HookPresets.merge(
|
|
4610
|
+
* HookPresets.monitoring(), // Includes errorLogging
|
|
4611
|
+
* customErrorAnalytics
|
|
4612
|
+
* ))
|
|
4613
|
+
* ```
|
|
4614
|
+
*
|
|
4615
|
+
* @example
|
|
4616
|
+
* ```typescript
|
|
4617
|
+
* // Error analytics collection
|
|
4618
|
+
* const errors: any[] = [];
|
|
4619
|
+
* .withHooks(HookPresets.merge(
|
|
4620
|
+
* HookPresets.errorLogging(),
|
|
4621
|
+
* {
|
|
4622
|
+
* observers: {
|
|
4623
|
+
* onLLMCallError: async (ctx) => {
|
|
4624
|
+
* errors.push({ type: 'llm', error: ctx.error, recovered: ctx.recovered });
|
|
4625
|
+
* },
|
|
4626
|
+
* },
|
|
4627
|
+
* }
|
|
4628
|
+
* ))
|
|
4629
|
+
* ```
|
|
4630
|
+
*
|
|
4631
|
+
* @see {@link https://github.com/zbigniewsobiecki/llmist/blob/main/docs/HOOKS.md#hookpresetserrorlogging | Full documentation}
|
|
4632
|
+
*/
|
|
4633
|
+
static errorLogging() {
|
|
4634
|
+
return {
|
|
4635
|
+
observers: {
|
|
4636
|
+
onLLMCallError: async (ctx) => {
|
|
4637
|
+
console.error(`\u274C LLM Error (iteration ${ctx.iteration}):`, ctx.error.message);
|
|
4638
|
+
console.error(` Model: ${ctx.options.model}`);
|
|
4639
|
+
console.error(` Recovered: ${ctx.recovered}`);
|
|
4640
|
+
},
|
|
4641
|
+
onGadgetExecutionComplete: async (ctx) => {
|
|
4642
|
+
if (ctx.error) {
|
|
4643
|
+
console.error(`\u274C Gadget Error: ${ctx.gadgetName}`);
|
|
4644
|
+
console.error(` Error: ${ctx.error}`);
|
|
4645
|
+
console.error(` Parameters:`, JSON.stringify(ctx.parameters, null, 2));
|
|
4646
|
+
}
|
|
4647
|
+
}
|
|
4648
|
+
}
|
|
4649
|
+
};
|
|
4650
|
+
}
|
|
4651
|
+
/**
|
|
4652
|
+
* Tracks context compaction events.
|
|
4653
|
+
*
|
|
4654
|
+
* **Output:**
|
|
4655
|
+
* - Compaction events with 🗜️ emoji
|
|
4656
|
+
* - Strategy name, tokens before/after, and savings
|
|
4657
|
+
* - Cumulative statistics
|
|
4658
|
+
*
|
|
4659
|
+
* **Use cases:**
|
|
4660
|
+
* - Monitoring long-running conversations
|
|
4661
|
+
* - Understanding when and how compaction occurs
|
|
4662
|
+
* - Debugging context management issues
|
|
4663
|
+
*
|
|
4664
|
+
* **Performance:** Minimal overhead. Simple console output.
|
|
4665
|
+
*
|
|
4666
|
+
* @returns Hook configuration that can be passed to .withHooks()
|
|
4667
|
+
*
|
|
4668
|
+
* @example
|
|
4669
|
+
* ```typescript
|
|
4670
|
+
* await LLMist.createAgent()
|
|
4671
|
+
* .withHooks(HookPresets.compactionTracking())
|
|
4672
|
+
* .ask("Your prompt");
|
|
4673
|
+
* ```
|
|
4674
|
+
*/
|
|
4675
|
+
static compactionTracking() {
|
|
4676
|
+
return {
|
|
4677
|
+
observers: {
|
|
4678
|
+
onCompaction: async (ctx) => {
|
|
4679
|
+
const saved = ctx.event.tokensBefore - ctx.event.tokensAfter;
|
|
4680
|
+
const percent = (saved / ctx.event.tokensBefore * 100).toFixed(1);
|
|
4681
|
+
console.log(
|
|
4682
|
+
`\u{1F5DC}\uFE0F Compaction (${ctx.event.strategy}): ${ctx.event.tokensBefore} \u2192 ${ctx.event.tokensAfter} tokens (saved ${saved}, ${percent}%)`
|
|
4683
|
+
);
|
|
4684
|
+
console.log(` Messages: ${ctx.event.messagesBefore} \u2192 ${ctx.event.messagesAfter}`);
|
|
4685
|
+
if (ctx.stats.totalCompactions > 1) {
|
|
4686
|
+
console.log(
|
|
4687
|
+
` Cumulative: ${ctx.stats.totalCompactions} compactions, ${ctx.stats.totalTokensSaved} tokens saved`
|
|
4688
|
+
);
|
|
4689
|
+
}
|
|
4690
|
+
}
|
|
4691
|
+
}
|
|
4692
|
+
};
|
|
4693
|
+
}
|
|
4694
|
+
/**
|
|
4695
|
+
* Logs LLM requests and responses to files for debugging and audit trails.
|
|
4696
|
+
*
|
|
4697
|
+
* Files are named `{counter}.request` and `{counter}.response` where counter
|
|
4698
|
+
* is a zero-padded number that increments with each LLM call.
|
|
4699
|
+
*
|
|
4700
|
+
* **Output:**
|
|
4701
|
+
* - Request files containing formatted LLM message history
|
|
4702
|
+
* - Response files containing raw LLM output
|
|
4703
|
+
*
|
|
4704
|
+
* **Use cases:**
|
|
4705
|
+
* - Debugging complex agent interactions
|
|
4706
|
+
* - Creating audit trails for compliance
|
|
4707
|
+
* - Analyzing LLM behavior patterns
|
|
4708
|
+
* - Replaying conversations for testing
|
|
4709
|
+
*
|
|
4710
|
+
* **Performance:** Minimal overhead - only file I/O, no synchronous blocking.
|
|
4711
|
+
*
|
|
4712
|
+
* **Note:** Can also be enabled via `LLMIST_LOG_RAW_DIRECTORY` environment
|
|
4713
|
+
* variable for zero-code activation.
|
|
4714
|
+
*
|
|
4715
|
+
* @param options - File logging options
|
|
4716
|
+
* @param options.directory - Directory where log files will be written
|
|
4717
|
+
* @param options.startingCounter - Starting counter (default: 1)
|
|
4718
|
+
* @param options.counterPadding - Number of digits for padding (default: 4)
|
|
4719
|
+
* @param options.skipSubagents - Skip subagent calls (default: true)
|
|
4720
|
+
* @param options.formatRequest - Custom request formatter
|
|
4721
|
+
* @param options.onFileWritten - Callback after each file is written
|
|
4722
|
+
* @returns Hook configuration that can be passed to .withHooks()
|
|
4723
|
+
*
|
|
4724
|
+
* @example
|
|
4725
|
+
* ```typescript
|
|
4726
|
+
* // Basic file logging
|
|
4727
|
+
* await LLMist.createAgent()
|
|
4728
|
+
* .withHooks(HookPresets.fileLogging({
|
|
4729
|
+
* directory: './debug-logs'
|
|
4730
|
+
* }))
|
|
4731
|
+
* .ask("Hello");
|
|
4732
|
+
* // Creates: ./debug-logs/0001.request
|
|
4733
|
+
* // ./debug-logs/0001.response
|
|
4734
|
+
* ```
|
|
4735
|
+
*
|
|
4736
|
+
* @example
|
|
4737
|
+
* ```typescript
|
|
4738
|
+
* // With callback for tracking
|
|
4739
|
+
* await LLMist.createAgent()
|
|
4740
|
+
* .withHooks(HookPresets.fileLogging({
|
|
4741
|
+
* directory: './logs',
|
|
4742
|
+
* onFileWritten: (info) => {
|
|
4743
|
+
* console.log(`Wrote ${info.type}: ${info.filePath}`);
|
|
4744
|
+
* }
|
|
4745
|
+
* }))
|
|
4746
|
+
* .ask("Hello");
|
|
4747
|
+
* ```
|
|
4748
|
+
*
|
|
4749
|
+
* @example
|
|
4750
|
+
* ```typescript
|
|
4751
|
+
* // Combined with other presets
|
|
4752
|
+
* .withHooks(HookPresets.merge(
|
|
4753
|
+
* HookPresets.fileLogging({ directory: logDir }),
|
|
4754
|
+
* HookPresets.progressTracking({ onProgress: updateUI }),
|
|
4755
|
+
* HookPresets.errorLogging()
|
|
4756
|
+
* ))
|
|
4757
|
+
* ```
|
|
4758
|
+
*
|
|
4759
|
+
* @see {@link https://github.com/zbigniewsobiecki/llmist/blob/main/docs/HOOKS.md#hookpresetsfileloggingoptions | Full documentation}
|
|
4760
|
+
*/
|
|
4761
|
+
static fileLogging(options) {
|
|
4762
|
+
return createFileLoggingHooks(options);
|
|
4763
|
+
}
|
|
4764
|
+
/**
|
|
4765
|
+
* Returns empty hook configuration for clean output without any logging.
|
|
4766
|
+
*
|
|
4767
|
+
* **Output:**
|
|
4768
|
+
* - None. Returns {} (empty object).
|
|
4769
|
+
*
|
|
4770
|
+
* **Use cases:**
|
|
4771
|
+
* - Clean test output without console noise
|
|
4772
|
+
* - Production environments where logging is handled externally
|
|
4773
|
+
* - Baseline for custom hook development
|
|
4774
|
+
* - Temporary disable of all hook output
|
|
4775
|
+
*
|
|
4776
|
+
* **Performance:** Zero overhead. No-op hook configuration.
|
|
4777
|
+
*
|
|
4778
|
+
* @returns Empty hook configuration
|
|
4779
|
+
*
|
|
4780
|
+
* @example
|
|
4781
|
+
* ```typescript
|
|
4782
|
+
* // Clean test output
|
|
4783
|
+
* describe('Agent tests', () => {
|
|
4784
|
+
* it('should calculate correctly', async () => {
|
|
4785
|
+
* const result = await LLMist.createAgent()
|
|
4786
|
+
* .withHooks(HookPresets.silent()) // No console output
|
|
4787
|
+
* .withGadgets(Calculator)
|
|
4788
|
+
* .askAndCollect("What is 15 times 23?");
|
|
4789
|
+
*
|
|
4790
|
+
* expect(result).toContain("345");
|
|
4791
|
+
* });
|
|
4792
|
+
* });
|
|
4793
|
+
* ```
|
|
4794
|
+
*
|
|
4795
|
+
* @example
|
|
4796
|
+
* ```typescript
|
|
4797
|
+
* // Conditional silence based on environment
|
|
4798
|
+
* const isTesting = process.env.NODE_ENV === 'test';
|
|
4799
|
+
* .withHooks(isTesting ? HookPresets.silent() : HookPresets.monitoring())
|
|
4800
|
+
* ```
|
|
4801
|
+
*
|
|
4802
|
+
* @see {@link https://github.com/zbigniewsobiecki/llmist/blob/main/docs/HOOKS.md#hookpresetssilent | Full documentation}
|
|
4803
|
+
*/
|
|
4804
|
+
static silent() {
|
|
4805
|
+
return {};
|
|
4806
|
+
}
|
|
4807
|
+
/**
|
|
4808
|
+
* Combines multiple hook configurations into one.
|
|
4809
|
+
*
|
|
4810
|
+
* Merge allows you to compose preset and custom hooks for modular monitoring
|
|
4811
|
+
* configurations. Understanding merge behavior is crucial for proper composition.
|
|
4812
|
+
*
|
|
4813
|
+
* **Merge behavior:**
|
|
4814
|
+
* - **Observers:** Composed - all handlers run sequentially in order
|
|
4815
|
+
* - **Interceptors:** Last one wins - only the last interceptor applies
|
|
4816
|
+
* - **Controllers:** Last one wins - only the last controller applies
|
|
4817
|
+
*
|
|
4818
|
+
* **Why interceptors/controllers don't compose:**
|
|
4819
|
+
* - Interceptors have different signatures per method, making composition impractical
|
|
4820
|
+
* - Controllers return specific actions that can't be meaningfully combined
|
|
4821
|
+
* - Only observers support composition because they're read-only and independent
|
|
4822
|
+
*
|
|
4823
|
+
* **Use cases:**
|
|
4824
|
+
* - Combining multiple presets (logging + timing + tokens)
|
|
4825
|
+
* - Adding custom hooks to presets
|
|
4826
|
+
* - Building modular, reusable monitoring configurations
|
|
4827
|
+
* - Environment-specific hook composition
|
|
4828
|
+
*
|
|
4829
|
+
* **Performance:** Minimal overhead for merging. Runtime performance depends on merged hooks.
|
|
4830
|
+
*
|
|
4831
|
+
* @param hookSets - Variable number of hook configurations to merge
|
|
4832
|
+
* @returns Single merged hook configuration with composed/overridden handlers
|
|
4833
|
+
*
|
|
4834
|
+
* @example
|
|
4835
|
+
* ```typescript
|
|
4836
|
+
* // Combine multiple presets
|
|
4837
|
+
* .withHooks(HookPresets.merge(
|
|
4838
|
+
* HookPresets.logging(),
|
|
4839
|
+
* HookPresets.timing(),
|
|
4840
|
+
* HookPresets.tokenTracking()
|
|
4841
|
+
* ))
|
|
4842
|
+
* // All observers from all three presets will run
|
|
4843
|
+
* ```
|
|
4844
|
+
*
|
|
4845
|
+
* @example
|
|
4846
|
+
* ```typescript
|
|
4847
|
+
* // Add custom observer to preset (both run)
|
|
4848
|
+
* .withHooks(HookPresets.merge(
|
|
4849
|
+
* HookPresets.timing(),
|
|
4850
|
+
* {
|
|
4851
|
+
* observers: {
|
|
4852
|
+
* onLLMCallComplete: async (ctx) => {
|
|
4853
|
+
* await saveMetrics({ tokens: ctx.usage?.totalTokens });
|
|
4854
|
+
* },
|
|
4855
|
+
* },
|
|
4856
|
+
* }
|
|
4857
|
+
* ))
|
|
4858
|
+
* ```
|
|
4859
|
+
*
|
|
4860
|
+
* @example
|
|
4861
|
+
* ```typescript
|
|
4862
|
+
* // Multiple interceptors (last wins!)
|
|
4863
|
+
* .withHooks(HookPresets.merge(
|
|
4864
|
+
* {
|
|
4865
|
+
* interceptors: {
|
|
4866
|
+
* interceptTextChunk: (chunk) => chunk.toUpperCase(), // Ignored
|
|
4867
|
+
* },
|
|
4868
|
+
* },
|
|
4869
|
+
* {
|
|
4870
|
+
* interceptors: {
|
|
4871
|
+
* interceptTextChunk: (chunk) => chunk.toLowerCase(), // This wins
|
|
4872
|
+
* },
|
|
4873
|
+
* }
|
|
4874
|
+
* ))
|
|
4875
|
+
* // Result: text will be lowercase
|
|
4876
|
+
* ```
|
|
4877
|
+
*
|
|
4878
|
+
* @example
|
|
4879
|
+
* ```typescript
|
|
4880
|
+
* // Modular environment-based configuration
|
|
4881
|
+
* const baseHooks = HookPresets.errorLogging();
|
|
4882
|
+
* const devHooks = HookPresets.merge(baseHooks, HookPresets.monitoring({ verbose: true }));
|
|
4883
|
+
* const prodHooks = HookPresets.merge(baseHooks, HookPresets.tokenTracking());
|
|
4884
|
+
*
|
|
4885
|
+
* const hooks = process.env.NODE_ENV === 'production' ? prodHooks : devHooks;
|
|
4886
|
+
* .withHooks(hooks)
|
|
4887
|
+
* ```
|
|
4888
|
+
*
|
|
4889
|
+
* @see {@link https://github.com/zbigniewsobiecki/llmist/blob/main/docs/HOOKS.md#hookpresetsmergehooksets | Full documentation}
|
|
4890
|
+
*/
|
|
4891
|
+
static merge(...hookSets) {
|
|
4892
|
+
const merged = {
|
|
4893
|
+
observers: {},
|
|
4894
|
+
interceptors: {},
|
|
4895
|
+
controllers: {}
|
|
4896
|
+
};
|
|
4897
|
+
for (const hooks of hookSets) {
|
|
4898
|
+
if (hooks.observers) {
|
|
4899
|
+
for (const [key, handler] of Object.entries(hooks.observers)) {
|
|
4900
|
+
const typedKey = key;
|
|
4901
|
+
if (merged.observers[typedKey]) {
|
|
4902
|
+
const existing = merged.observers[typedKey];
|
|
4903
|
+
merged.observers[typedKey] = async (ctx) => {
|
|
4904
|
+
await existing(ctx);
|
|
4905
|
+
await handler(ctx);
|
|
4906
|
+
};
|
|
4907
|
+
} else {
|
|
4908
|
+
merged.observers[typedKey] = handler;
|
|
4909
|
+
}
|
|
4910
|
+
}
|
|
4911
|
+
}
|
|
4912
|
+
if (hooks.interceptors) {
|
|
4913
|
+
Object.assign(merged.interceptors, hooks.interceptors);
|
|
4914
|
+
}
|
|
4915
|
+
if (hooks.controllers) {
|
|
4916
|
+
Object.assign(merged.controllers, hooks.controllers);
|
|
4917
|
+
}
|
|
4918
|
+
}
|
|
4919
|
+
return merged;
|
|
3962
4920
|
}
|
|
3963
4921
|
/**
|
|
3964
|
-
*
|
|
4922
|
+
* Composite preset combining logging, timing, tokenTracking, and errorLogging.
|
|
3965
4923
|
*
|
|
3966
|
-
*
|
|
3967
|
-
*
|
|
4924
|
+
* This is the recommended preset for development and initial production deployments,
|
|
4925
|
+
* providing comprehensive observability with a single method call.
|
|
4926
|
+
*
|
|
4927
|
+
* **Includes:**
|
|
4928
|
+
* - All output from `logging()` preset (with optional verbosity)
|
|
4929
|
+
* - All output from `timing()` preset (execution times)
|
|
4930
|
+
* - All output from `tokenTracking()` preset (token usage)
|
|
4931
|
+
* - All output from `errorLogging()` preset (error details)
|
|
4932
|
+
*
|
|
4933
|
+
* **Output format:**
|
|
4934
|
+
* - Event logging: [LLM]/[GADGET] messages
|
|
4935
|
+
* - Timing: ⏱️ emoji with milliseconds
|
|
4936
|
+
* - Tokens: 📊 emoji with per-call and cumulative counts
|
|
4937
|
+
* - Errors: ❌ emoji with full error details
|
|
4938
|
+
*
|
|
4939
|
+
* **Use cases:**
|
|
4940
|
+
* - Full observability during development
|
|
4941
|
+
* - Comprehensive monitoring in production
|
|
4942
|
+
* - One-liner for complete agent visibility
|
|
4943
|
+
* - Troubleshooting and debugging with full context
|
|
4944
|
+
*
|
|
4945
|
+
* **Performance:** Combined overhead of all four presets, but still minimal in practice.
|
|
4946
|
+
*
|
|
4947
|
+
* @param options - Monitoring options
|
|
4948
|
+
* @param options.verbose - Passed to logging() preset for detailed output. Default: false
|
|
4949
|
+
* @returns Merged hook configuration combining all monitoring presets
|
|
3968
4950
|
*
|
|
3969
4951
|
* @example
|
|
3970
4952
|
* ```typescript
|
|
3971
|
-
*
|
|
3972
|
-
*
|
|
4953
|
+
* // Basic monitoring (recommended for development)
|
|
4954
|
+
* await LLMist.createAgent()
|
|
4955
|
+
* .withHooks(HookPresets.monitoring())
|
|
4956
|
+
* .withGadgets(Calculator, Weather)
|
|
4957
|
+
* .ask("What is 15 times 23, and what's the weather in NYC?");
|
|
4958
|
+
* // Output: All events, timing, tokens, and errors in one place
|
|
4959
|
+
* ```
|
|
4960
|
+
*
|
|
4961
|
+
* @example
|
|
4962
|
+
* ```typescript
|
|
4963
|
+
* // Verbose monitoring with full details
|
|
4964
|
+
* await LLMist.createAgent()
|
|
4965
|
+
* .withHooks(HookPresets.monitoring({ verbose: true }))
|
|
4966
|
+
* .ask("Your prompt");
|
|
4967
|
+
* // Output includes: parameters, results, and complete responses
|
|
4968
|
+
* ```
|
|
4969
|
+
*
|
|
4970
|
+
* @example
|
|
4971
|
+
* ```typescript
|
|
4972
|
+
* // Environment-based monitoring
|
|
4973
|
+
* const isDev = process.env.NODE_ENV === 'development';
|
|
4974
|
+
* .withHooks(HookPresets.monitoring({ verbose: isDev }))
|
|
3973
4975
|
* ```
|
|
4976
|
+
*
|
|
4977
|
+
* @see {@link https://github.com/zbigniewsobiecki/llmist/blob/main/docs/HOOKS.md#hookpresetsmonitoringoptions | Full documentation}
|
|
3974
4978
|
*/
|
|
3975
|
-
|
|
3976
|
-
|
|
3977
|
-
|
|
3978
|
-
|
|
3979
|
-
|
|
3980
|
-
|
|
3981
|
-
|
|
3982
|
-
// Register a gadget by name
|
|
3983
|
-
register(name, gadget) {
|
|
3984
|
-
const normalizedName = name.toLowerCase();
|
|
3985
|
-
if (this.gadgets.has(normalizedName)) {
|
|
3986
|
-
throw new Error(`Gadget '${name}' is already registered`);
|
|
3987
|
-
}
|
|
3988
|
-
if (gadget.parameterSchema) {
|
|
3989
|
-
validateGadgetSchema(gadget.parameterSchema, name);
|
|
3990
|
-
}
|
|
3991
|
-
this.gadgets.set(normalizedName, gadget);
|
|
3992
|
-
}
|
|
3993
|
-
// Register a gadget using its name property or class name
|
|
3994
|
-
registerByClass(gadget) {
|
|
3995
|
-
const name = gadget.name ?? gadget.constructor.name;
|
|
3996
|
-
this.register(name, gadget);
|
|
3997
|
-
}
|
|
3998
|
-
// Get gadget by name (case-insensitive)
|
|
3999
|
-
get(name) {
|
|
4000
|
-
return this.gadgets.get(name.toLowerCase());
|
|
4001
|
-
}
|
|
4002
|
-
// Check if gadget exists (case-insensitive)
|
|
4003
|
-
has(name) {
|
|
4004
|
-
return this.gadgets.has(name.toLowerCase());
|
|
4005
|
-
}
|
|
4006
|
-
// Get all registered gadget names
|
|
4007
|
-
getNames() {
|
|
4008
|
-
return Array.from(this.gadgets.keys());
|
|
4009
|
-
}
|
|
4010
|
-
// Get all gadgets for instruction generation
|
|
4011
|
-
getAll() {
|
|
4012
|
-
return Array.from(this.gadgets.values());
|
|
4013
|
-
}
|
|
4014
|
-
// Unregister gadget (useful for testing, case-insensitive)
|
|
4015
|
-
unregister(name) {
|
|
4016
|
-
return this.gadgets.delete(name.toLowerCase());
|
|
4017
|
-
}
|
|
4018
|
-
// Clear all gadgets (useful for testing)
|
|
4019
|
-
clear() {
|
|
4020
|
-
this.gadgets.clear();
|
|
4979
|
+
static monitoring(options = {}) {
|
|
4980
|
+
return _HookPresets.merge(
|
|
4981
|
+
_HookPresets.logging(options),
|
|
4982
|
+
_HookPresets.timing(),
|
|
4983
|
+
_HookPresets.tokenTracking(),
|
|
4984
|
+
_HookPresets.errorLogging()
|
|
4985
|
+
);
|
|
4021
4986
|
}
|
|
4022
4987
|
};
|
|
4023
4988
|
}
|
|
@@ -9213,6 +10178,8 @@ var init_builder = __esm({
|
|
|
9213
10178
|
init_agent();
|
|
9214
10179
|
init_agent_internal_key();
|
|
9215
10180
|
init_event_handlers();
|
|
10181
|
+
init_file_logging();
|
|
10182
|
+
init_hook_presets();
|
|
9216
10183
|
AgentBuilder = class {
|
|
9217
10184
|
client;
|
|
9218
10185
|
model;
|
|
@@ -10006,9 +10973,16 @@ ${endPrefix}`
|
|
|
10006
10973
|
* Note: Subagent event visibility is now handled entirely by the ExecutionTree.
|
|
10007
10974
|
* When a subagent uses withParentContext(ctx), it shares the parent's tree,
|
|
10008
10975
|
* and all events are automatically visible to tree subscribers (like the TUI).
|
|
10976
|
+
*
|
|
10977
|
+
* Environment-based file logging (via LLMIST_LOG_RAW_DIRECTORY) is automatically
|
|
10978
|
+
* injected if the env var is set. User-provided hooks take precedence.
|
|
10009
10979
|
*/
|
|
10010
10980
|
composeHooks() {
|
|
10011
|
-
|
|
10981
|
+
let hooks = this.hooks;
|
|
10982
|
+
const envFileLogging = getEnvFileLoggingHooks();
|
|
10983
|
+
if (envFileLogging) {
|
|
10984
|
+
hooks = hooks ? HookPresets.merge(envFileLogging, hooks) : envFileLogging;
|
|
10985
|
+
}
|
|
10012
10986
|
if (!this.trailingMessage) {
|
|
10013
10987
|
return hooks;
|
|
10014
10988
|
}
|
|
@@ -13953,9 +14927,11 @@ __export(index_exports, {
|
|
|
13953
14927
|
filterRootEvents: () => filterRootEvents,
|
|
13954
14928
|
format: () => format,
|
|
13955
14929
|
formatBytes: () => formatBytes,
|
|
14930
|
+
formatCallNumber: () => formatCallNumber,
|
|
13956
14931
|
formatDate: () => formatDate,
|
|
13957
14932
|
formatDuration: () => formatDuration,
|
|
13958
14933
|
formatLLMError: () => formatLLMError,
|
|
14934
|
+
formatLlmRequest: () => formatLlmRequest,
|
|
13959
14935
|
gadgetError: () => gadgetError,
|
|
13960
14936
|
gadgetSuccess: () => gadgetSuccess,
|
|
13961
14937
|
getErrorMessage: () => getErrorMessage,
|
|
@@ -14027,781 +15003,8 @@ var import_zod3 = require("zod");
|
|
|
14027
15003
|
init_agent();
|
|
14028
15004
|
init_builder();
|
|
14029
15005
|
init_event_handlers();
|
|
14030
|
-
|
|
14031
|
-
|
|
14032
|
-
var HookPresets = class _HookPresets {
|
|
14033
|
-
/**
|
|
14034
|
-
* Logs LLM calls and gadget execution to console with optional verbosity.
|
|
14035
|
-
*
|
|
14036
|
-
* **Output (basic mode):**
|
|
14037
|
-
* - LLM call start/complete events with iteration numbers
|
|
14038
|
-
* - Gadget execution start/complete with gadget names
|
|
14039
|
-
* - Token counts when available
|
|
14040
|
-
*
|
|
14041
|
-
* **Output (verbose mode):**
|
|
14042
|
-
* - All basic mode output
|
|
14043
|
-
* - Full gadget parameters (formatted JSON)
|
|
14044
|
-
* - Full gadget results
|
|
14045
|
-
* - Complete LLM response text
|
|
14046
|
-
*
|
|
14047
|
-
* **Use cases:**
|
|
14048
|
-
* - Basic development debugging and execution flow visibility
|
|
14049
|
-
* - Understanding agent decision-making and tool usage
|
|
14050
|
-
* - Troubleshooting gadget invocations
|
|
14051
|
-
*
|
|
14052
|
-
* **Performance:** Minimal overhead. Console writes are synchronous but fast.
|
|
14053
|
-
*
|
|
14054
|
-
* @param options - Logging options
|
|
14055
|
-
* @param options.verbose - Include full parameters and results. Default: false
|
|
14056
|
-
* @returns Hook configuration that can be passed to .withHooks()
|
|
14057
|
-
*
|
|
14058
|
-
* @example
|
|
14059
|
-
* ```typescript
|
|
14060
|
-
* // Basic logging
|
|
14061
|
-
* await LLMist.createAgent()
|
|
14062
|
-
* .withHooks(HookPresets.logging())
|
|
14063
|
-
* .ask("Calculate 15 * 23");
|
|
14064
|
-
* // Output: [LLM] Starting call (iteration 0)
|
|
14065
|
-
* // [GADGET] Executing Calculator
|
|
14066
|
-
* // [GADGET] Completed Calculator
|
|
14067
|
-
* // [LLM] Completed (tokens: 245)
|
|
14068
|
-
* ```
|
|
14069
|
-
*
|
|
14070
|
-
* @example
|
|
14071
|
-
* ```typescript
|
|
14072
|
-
* // Verbose logging with full details
|
|
14073
|
-
* await LLMist.createAgent()
|
|
14074
|
-
* .withHooks(HookPresets.logging({ verbose: true }))
|
|
14075
|
-
* .ask("Calculate 15 * 23");
|
|
14076
|
-
* // Output includes: parameters, results, and full responses
|
|
14077
|
-
* ```
|
|
14078
|
-
*
|
|
14079
|
-
* @example
|
|
14080
|
-
* ```typescript
|
|
14081
|
-
* // Environment-based verbosity
|
|
14082
|
-
* const isDev = process.env.NODE_ENV === 'development';
|
|
14083
|
-
* .withHooks(HookPresets.logging({ verbose: isDev }))
|
|
14084
|
-
* ```
|
|
14085
|
-
*
|
|
14086
|
-
* @see {@link https://github.com/zbigniewsobiecki/llmist/blob/main/docs/HOOKS.md#hookpresetsloggingoptions | Full documentation}
|
|
14087
|
-
*/
|
|
14088
|
-
static logging(options = {}) {
|
|
14089
|
-
return {
|
|
14090
|
-
observers: {
|
|
14091
|
-
onLLMCallStart: async (ctx) => {
|
|
14092
|
-
console.log(`[LLM] Starting call (iteration ${ctx.iteration})`);
|
|
14093
|
-
},
|
|
14094
|
-
onLLMCallComplete: async (ctx) => {
|
|
14095
|
-
const tokens = ctx.usage?.totalTokens ?? "unknown";
|
|
14096
|
-
console.log(`[LLM] Completed (tokens: ${tokens})`);
|
|
14097
|
-
if (options.verbose && ctx.finalMessage) {
|
|
14098
|
-
console.log(`[LLM] Response: ${ctx.finalMessage}`);
|
|
14099
|
-
}
|
|
14100
|
-
},
|
|
14101
|
-
onGadgetExecutionStart: async (ctx) => {
|
|
14102
|
-
console.log(`[GADGET] Executing ${ctx.gadgetName}`);
|
|
14103
|
-
if (options.verbose) {
|
|
14104
|
-
console.log(`[GADGET] Parameters:`, JSON.stringify(ctx.parameters, null, 2));
|
|
14105
|
-
}
|
|
14106
|
-
},
|
|
14107
|
-
onGadgetExecutionComplete: async (ctx) => {
|
|
14108
|
-
console.log(`[GADGET] Completed ${ctx.gadgetName}`);
|
|
14109
|
-
if (options.verbose) {
|
|
14110
|
-
const display = ctx.error ?? ctx.finalResult ?? "(no result)";
|
|
14111
|
-
console.log(`[GADGET] Result: ${display}`);
|
|
14112
|
-
}
|
|
14113
|
-
}
|
|
14114
|
-
}
|
|
14115
|
-
};
|
|
14116
|
-
}
|
|
14117
|
-
/**
|
|
14118
|
-
* Measures and logs execution time for LLM calls and gadgets.
|
|
14119
|
-
*
|
|
14120
|
-
* **Output:**
|
|
14121
|
-
* - Duration in milliseconds with ⏱️ emoji for each operation
|
|
14122
|
-
* - Separate timing for each LLM iteration
|
|
14123
|
-
* - Separate timing for each gadget execution
|
|
14124
|
-
*
|
|
14125
|
-
* **Use cases:**
|
|
14126
|
-
* - Performance profiling and optimization
|
|
14127
|
-
* - Identifying slow operations (LLM calls vs gadget execution)
|
|
14128
|
-
* - Monitoring response times in production
|
|
14129
|
-
* - Capacity planning and SLA tracking
|
|
14130
|
-
*
|
|
14131
|
-
* **Performance:** Negligible overhead. Uses Date.now() for timing measurements.
|
|
14132
|
-
*
|
|
14133
|
-
* @returns Hook configuration that can be passed to .withHooks()
|
|
14134
|
-
*
|
|
14135
|
-
* @example
|
|
14136
|
-
* ```typescript
|
|
14137
|
-
* // Basic timing
|
|
14138
|
-
* await LLMist.createAgent()
|
|
14139
|
-
* .withHooks(HookPresets.timing())
|
|
14140
|
-
* .withGadgets(Weather, Database)
|
|
14141
|
-
* .ask("What's the weather in NYC?");
|
|
14142
|
-
* // Output: ⏱️ LLM call took 1234ms
|
|
14143
|
-
* // ⏱️ Gadget Weather took 567ms
|
|
14144
|
-
* // ⏱️ LLM call took 890ms
|
|
14145
|
-
* ```
|
|
14146
|
-
*
|
|
14147
|
-
* @example
|
|
14148
|
-
* ```typescript
|
|
14149
|
-
* // Combined with logging for full context
|
|
14150
|
-
* .withHooks(HookPresets.merge(
|
|
14151
|
-
* HookPresets.logging(),
|
|
14152
|
-
* HookPresets.timing()
|
|
14153
|
-
* ))
|
|
14154
|
-
* ```
|
|
14155
|
-
*
|
|
14156
|
-
* @example
|
|
14157
|
-
* ```typescript
|
|
14158
|
-
* // Correlate performance with cost
|
|
14159
|
-
* .withHooks(HookPresets.merge(
|
|
14160
|
-
* HookPresets.timing(),
|
|
14161
|
-
* HookPresets.tokenTracking()
|
|
14162
|
-
* ))
|
|
14163
|
-
* ```
|
|
14164
|
-
*
|
|
14165
|
-
* @see {@link https://github.com/zbigniewsobiecki/llmist/blob/main/docs/HOOKS.md#hookpresetstiming | Full documentation}
|
|
14166
|
-
*/
|
|
14167
|
-
static timing() {
|
|
14168
|
-
const timings = /* @__PURE__ */ new Map();
|
|
14169
|
-
return {
|
|
14170
|
-
observers: {
|
|
14171
|
-
onLLMCallStart: async (ctx) => {
|
|
14172
|
-
timings.set(`llm-${ctx.iteration}`, Date.now());
|
|
14173
|
-
},
|
|
14174
|
-
onLLMCallComplete: async (ctx) => {
|
|
14175
|
-
const start = timings.get(`llm-${ctx.iteration}`);
|
|
14176
|
-
if (start) {
|
|
14177
|
-
const duration = Date.now() - start;
|
|
14178
|
-
console.log(`\u23F1\uFE0F LLM call took ${duration}ms`);
|
|
14179
|
-
timings.delete(`llm-${ctx.iteration}`);
|
|
14180
|
-
}
|
|
14181
|
-
},
|
|
14182
|
-
onGadgetExecutionStart: async (ctx) => {
|
|
14183
|
-
const key = `gadget-${ctx.gadgetName}-${Date.now()}`;
|
|
14184
|
-
timings.set(key, Date.now());
|
|
14185
|
-
ctx._timingKey = key;
|
|
14186
|
-
},
|
|
14187
|
-
onGadgetExecutionComplete: async (ctx) => {
|
|
14188
|
-
const key = ctx._timingKey;
|
|
14189
|
-
if (key) {
|
|
14190
|
-
const start = timings.get(key);
|
|
14191
|
-
if (start) {
|
|
14192
|
-
const duration = Date.now() - start;
|
|
14193
|
-
console.log(`\u23F1\uFE0F Gadget ${ctx.gadgetName} took ${duration}ms`);
|
|
14194
|
-
timings.delete(key);
|
|
14195
|
-
}
|
|
14196
|
-
}
|
|
14197
|
-
}
|
|
14198
|
-
}
|
|
14199
|
-
};
|
|
14200
|
-
}
|
|
14201
|
-
/**
|
|
14202
|
-
* Tracks cumulative token usage across all LLM calls.
|
|
14203
|
-
*
|
|
14204
|
-
* **Output:**
|
|
14205
|
-
* - Per-call token count with 📊 emoji
|
|
14206
|
-
* - Cumulative total across all calls
|
|
14207
|
-
* - Call count for average calculations
|
|
14208
|
-
*
|
|
14209
|
-
* **Use cases:**
|
|
14210
|
-
* - Cost monitoring and budget tracking
|
|
14211
|
-
* - Optimizing prompts to reduce token usage
|
|
14212
|
-
* - Comparing token efficiency across different approaches
|
|
14213
|
-
* - Real-time cost estimation
|
|
14214
|
-
*
|
|
14215
|
-
* **Performance:** Minimal overhead. Simple counter increments.
|
|
14216
|
-
*
|
|
14217
|
-
* **Note:** Token counts depend on the provider's response. Some providers
|
|
14218
|
-
* may not include usage data, in which case counts won't be logged.
|
|
14219
|
-
*
|
|
14220
|
-
* @returns Hook configuration that can be passed to .withHooks()
|
|
14221
|
-
*
|
|
14222
|
-
* @example
|
|
14223
|
-
* ```typescript
|
|
14224
|
-
* // Basic token tracking
|
|
14225
|
-
* await LLMist.createAgent()
|
|
14226
|
-
* .withHooks(HookPresets.tokenTracking())
|
|
14227
|
-
* .ask("Summarize this document...");
|
|
14228
|
-
* // Output: 📊 Tokens this call: 1,234
|
|
14229
|
-
* // 📊 Total tokens: 1,234 (across 1 calls)
|
|
14230
|
-
* // 📊 Tokens this call: 567
|
|
14231
|
-
* // 📊 Total tokens: 1,801 (across 2 calls)
|
|
14232
|
-
* ```
|
|
14233
|
-
*
|
|
14234
|
-
* @example
|
|
14235
|
-
* ```typescript
|
|
14236
|
-
* // Cost calculation with custom hook
|
|
14237
|
-
* let totalTokens = 0;
|
|
14238
|
-
* .withHooks(HookPresets.merge(
|
|
14239
|
-
* HookPresets.tokenTracking(),
|
|
14240
|
-
* {
|
|
14241
|
-
* observers: {
|
|
14242
|
-
* onLLMCallComplete: async (ctx) => {
|
|
14243
|
-
* totalTokens += ctx.usage?.totalTokens ?? 0;
|
|
14244
|
-
* const cost = (totalTokens / 1_000_000) * 3.0; // $3 per 1M tokens
|
|
14245
|
-
* console.log(`💰 Estimated cost: $${cost.toFixed(4)}`);
|
|
14246
|
-
* },
|
|
14247
|
-
* },
|
|
14248
|
-
* }
|
|
14249
|
-
* ))
|
|
14250
|
-
* ```
|
|
14251
|
-
*
|
|
14252
|
-
* @see {@link https://github.com/zbigniewsobiecki/llmist/blob/main/docs/HOOKS.md#hookpresetstokentracking | Full documentation}
|
|
14253
|
-
*/
|
|
14254
|
-
static tokenTracking() {
|
|
14255
|
-
let totalTokens = 0;
|
|
14256
|
-
let totalCalls = 0;
|
|
14257
|
-
return {
|
|
14258
|
-
observers: {
|
|
14259
|
-
onLLMCallComplete: async (ctx) => {
|
|
14260
|
-
totalCalls++;
|
|
14261
|
-
if (ctx.usage?.totalTokens) {
|
|
14262
|
-
totalTokens += ctx.usage.totalTokens;
|
|
14263
|
-
console.log(`\u{1F4CA} Tokens this call: ${ctx.usage.totalTokens}`);
|
|
14264
|
-
console.log(`\u{1F4CA} Total tokens: ${totalTokens} (across ${totalCalls} calls)`);
|
|
14265
|
-
}
|
|
14266
|
-
}
|
|
14267
|
-
}
|
|
14268
|
-
};
|
|
14269
|
-
}
|
|
14270
|
-
/**
|
|
14271
|
-
* Tracks comprehensive progress metrics including iterations, tokens, cost, and timing.
|
|
14272
|
-
*
|
|
14273
|
-
* **This preset showcases llmist's core capabilities by demonstrating:**
|
|
14274
|
-
* - Observer pattern for non-intrusive monitoring
|
|
14275
|
-
* - Integration with ModelRegistry for cost estimation
|
|
14276
|
-
* - Callback-based architecture for flexible UI updates
|
|
14277
|
-
* - Provider-agnostic token and cost tracking
|
|
14278
|
-
*
|
|
14279
|
-
* Unlike `tokenTracking()` which only logs to console, this preset provides
|
|
14280
|
-
* structured data through callbacks, making it perfect for building custom UIs,
|
|
14281
|
-
* dashboards, or progress indicators (like the llmist CLI).
|
|
14282
|
-
*
|
|
14283
|
-
* **Output (when logProgress: true):**
|
|
14284
|
-
* - Iteration number and call count
|
|
14285
|
-
* - Cumulative token usage (input + output)
|
|
14286
|
-
* - Cumulative cost in USD (requires modelRegistry)
|
|
14287
|
-
* - Elapsed time in seconds
|
|
14288
|
-
*
|
|
14289
|
-
* **Use cases:**
|
|
14290
|
-
* - Building CLI progress indicators with live updates
|
|
14291
|
-
* - Creating web dashboards with real-time metrics
|
|
14292
|
-
* - Budget monitoring and cost alerts
|
|
14293
|
-
* - Performance tracking and optimization
|
|
14294
|
-
* - Custom logging to external systems (Datadog, CloudWatch, etc.)
|
|
14295
|
-
*
|
|
14296
|
-
* **Performance:** Minimal overhead. Uses Date.now() for timing and optional
|
|
14297
|
-
* ModelRegistry.estimateCost() which is O(1) lookup. Callback invocation is
|
|
14298
|
-
* synchronous and fast.
|
|
14299
|
-
*
|
|
14300
|
-
* @param options - Progress tracking options
|
|
14301
|
-
* @param options.modelRegistry - ModelRegistry for cost estimation (optional)
|
|
14302
|
-
* @param options.onProgress - Callback invoked after each LLM call (optional)
|
|
14303
|
-
* @param options.logProgress - Log progress to console (default: false)
|
|
14304
|
-
* @returns Hook configuration with progress tracking observers
|
|
14305
|
-
*
|
|
14306
|
-
* @example
|
|
14307
|
-
* ```typescript
|
|
14308
|
-
* // Basic usage with callback (RECOMMENDED - used by llmist CLI)
|
|
14309
|
-
* import { LLMist, HookPresets } from 'llmist';
|
|
14310
|
-
*
|
|
14311
|
-
* const client = LLMist.create();
|
|
14312
|
-
*
|
|
14313
|
-
* await client.agent()
|
|
14314
|
-
* .withHooks(HookPresets.progressTracking({
|
|
14315
|
-
* modelRegistry: client.modelRegistry,
|
|
14316
|
-
* onProgress: (stats) => {
|
|
14317
|
-
* // Update your UI with stats
|
|
14318
|
-
* console.log(`#${stats.currentIteration} | ${stats.totalTokens} tokens | $${stats.totalCost.toFixed(4)}`);
|
|
14319
|
-
* }
|
|
14320
|
-
* }))
|
|
14321
|
-
* .withGadgets(Calculator)
|
|
14322
|
-
* .ask("Calculate 15 * 23");
|
|
14323
|
-
* // Output: #1 | 245 tokens | $0.0012
|
|
14324
|
-
* ```
|
|
14325
|
-
*
|
|
14326
|
-
* @example
|
|
14327
|
-
* ```typescript
|
|
14328
|
-
* // Console logging mode (quick debugging)
|
|
14329
|
-
* await client.agent()
|
|
14330
|
-
* .withHooks(HookPresets.progressTracking({
|
|
14331
|
-
* modelRegistry: client.modelRegistry,
|
|
14332
|
-
* logProgress: true // Simple console output
|
|
14333
|
-
* }))
|
|
14334
|
-
* .ask("Your prompt");
|
|
14335
|
-
* // Output: 📊 Progress: Iteration #1 | 245 tokens | $0.0012 | 1.2s
|
|
14336
|
-
* ```
|
|
14337
|
-
*
|
|
14338
|
-
* @example
|
|
14339
|
-
* ```typescript
|
|
14340
|
-
* // Budget monitoring with alerts
|
|
14341
|
-
* const BUDGET_USD = 0.10;
|
|
14342
|
-
*
|
|
14343
|
-
* await client.agent()
|
|
14344
|
-
* .withHooks(HookPresets.progressTracking({
|
|
14345
|
-
* modelRegistry: client.modelRegistry,
|
|
14346
|
-
* onProgress: (stats) => {
|
|
14347
|
-
* if (stats.totalCost > BUDGET_USD) {
|
|
14348
|
-
* throw new Error(`Budget exceeded: $${stats.totalCost.toFixed(4)}`);
|
|
14349
|
-
* }
|
|
14350
|
-
* }
|
|
14351
|
-
* }))
|
|
14352
|
-
* .ask("Long running task...");
|
|
14353
|
-
* ```
|
|
14354
|
-
*
|
|
14355
|
-
* @example
|
|
14356
|
-
* ```typescript
|
|
14357
|
-
* // Web dashboard integration
|
|
14358
|
-
* let progressBar: HTMLElement;
|
|
14359
|
-
*
|
|
14360
|
-
* await client.agent()
|
|
14361
|
-
* .withHooks(HookPresets.progressTracking({
|
|
14362
|
-
* modelRegistry: client.modelRegistry,
|
|
14363
|
-
* onProgress: (stats) => {
|
|
14364
|
-
* // Update web UI in real-time
|
|
14365
|
-
* progressBar.textContent = `Iteration ${stats.currentIteration}`;
|
|
14366
|
-
* progressBar.dataset.cost = stats.totalCost.toFixed(4);
|
|
14367
|
-
* progressBar.dataset.tokens = stats.totalTokens.toString();
|
|
14368
|
-
* }
|
|
14369
|
-
* }))
|
|
14370
|
-
* .ask("Your prompt");
|
|
14371
|
-
* ```
|
|
14372
|
-
*
|
|
14373
|
-
* @example
|
|
14374
|
-
* ```typescript
|
|
14375
|
-
* // External logging (Datadog, CloudWatch, etc.)
|
|
14376
|
-
* await client.agent()
|
|
14377
|
-
* .withHooks(HookPresets.progressTracking({
|
|
14378
|
-
* modelRegistry: client.modelRegistry,
|
|
14379
|
-
* onProgress: async (stats) => {
|
|
14380
|
-
* await metrics.gauge('llm.iteration', stats.currentIteration);
|
|
14381
|
-
* await metrics.gauge('llm.cost', stats.totalCost);
|
|
14382
|
-
* await metrics.gauge('llm.tokens', stats.totalTokens);
|
|
14383
|
-
* }
|
|
14384
|
-
* }))
|
|
14385
|
-
* .ask("Your prompt");
|
|
14386
|
-
* ```
|
|
14387
|
-
*
|
|
14388
|
-
* @see {@link https://github.com/zbigniewsobiecki/llmist/blob/main/docs/HOOKS.md#hookpresetsprogresstrackingoptions | Full documentation}
|
|
14389
|
-
* @see {@link ProgressTrackingOptions} for detailed options
|
|
14390
|
-
* @see {@link ProgressStats} for the callback data structure
|
|
14391
|
-
*/
|
|
14392
|
-
static progressTracking(options) {
|
|
14393
|
-
const { modelRegistry, onProgress, logProgress = false } = options ?? {};
|
|
14394
|
-
let totalCalls = 0;
|
|
14395
|
-
let currentIteration = 0;
|
|
14396
|
-
let totalInputTokens = 0;
|
|
14397
|
-
let totalOutputTokens = 0;
|
|
14398
|
-
let totalCost = 0;
|
|
14399
|
-
let totalGadgetCost = 0;
|
|
14400
|
-
const startTime = Date.now();
|
|
14401
|
-
return {
|
|
14402
|
-
observers: {
|
|
14403
|
-
// Track iteration on each LLM call start
|
|
14404
|
-
onLLMCallStart: async (ctx) => {
|
|
14405
|
-
currentIteration++;
|
|
14406
|
-
},
|
|
14407
|
-
// Accumulate metrics and report progress on each LLM call completion
|
|
14408
|
-
onLLMCallComplete: async (ctx) => {
|
|
14409
|
-
totalCalls++;
|
|
14410
|
-
if (ctx.usage) {
|
|
14411
|
-
totalInputTokens += ctx.usage.inputTokens;
|
|
14412
|
-
totalOutputTokens += ctx.usage.outputTokens;
|
|
14413
|
-
if (modelRegistry) {
|
|
14414
|
-
try {
|
|
14415
|
-
const modelName = ctx.options.model.includes(":") ? ctx.options.model.split(":")[1] : ctx.options.model;
|
|
14416
|
-
const costEstimate = modelRegistry.estimateCost(
|
|
14417
|
-
modelName,
|
|
14418
|
-
ctx.usage.inputTokens,
|
|
14419
|
-
ctx.usage.outputTokens
|
|
14420
|
-
);
|
|
14421
|
-
if (costEstimate) {
|
|
14422
|
-
totalCost += costEstimate.totalCost;
|
|
14423
|
-
}
|
|
14424
|
-
} catch (error) {
|
|
14425
|
-
if (logProgress) {
|
|
14426
|
-
console.warn(`\u26A0\uFE0F Cost estimation failed:`, error);
|
|
14427
|
-
}
|
|
14428
|
-
}
|
|
14429
|
-
}
|
|
14430
|
-
}
|
|
14431
|
-
const stats = {
|
|
14432
|
-
currentIteration,
|
|
14433
|
-
totalCalls,
|
|
14434
|
-
totalInputTokens,
|
|
14435
|
-
totalOutputTokens,
|
|
14436
|
-
totalTokens: totalInputTokens + totalOutputTokens,
|
|
14437
|
-
totalCost: totalCost + totalGadgetCost,
|
|
14438
|
-
elapsedSeconds: Number(((Date.now() - startTime) / 1e3).toFixed(1))
|
|
14439
|
-
};
|
|
14440
|
-
if (onProgress) {
|
|
14441
|
-
onProgress(stats);
|
|
14442
|
-
}
|
|
14443
|
-
if (logProgress) {
|
|
14444
|
-
const formattedTokens = stats.totalTokens >= 1e3 ? `${(stats.totalTokens / 1e3).toFixed(1)}k` : `${stats.totalTokens}`;
|
|
14445
|
-
const formattedCost = stats.totalCost > 0 ? `$${stats.totalCost.toFixed(4)}` : "$0";
|
|
14446
|
-
console.log(
|
|
14447
|
-
`\u{1F4CA} Progress: Iteration #${stats.currentIteration} | ${formattedTokens} tokens | ${formattedCost} | ${stats.elapsedSeconds}s`
|
|
14448
|
-
);
|
|
14449
|
-
}
|
|
14450
|
-
},
|
|
14451
|
-
// Track gadget execution costs
|
|
14452
|
-
onGadgetExecutionComplete: async (ctx) => {
|
|
14453
|
-
if (ctx.cost && ctx.cost > 0) {
|
|
14454
|
-
totalGadgetCost += ctx.cost;
|
|
14455
|
-
}
|
|
14456
|
-
}
|
|
14457
|
-
}
|
|
14458
|
-
};
|
|
14459
|
-
}
|
|
14460
|
-
/**
|
|
14461
|
-
* Logs detailed error information for debugging and troubleshooting.
|
|
14462
|
-
*
|
|
14463
|
-
* **Output:**
|
|
14464
|
-
* - LLM errors with ❌ emoji, including model and recovery status
|
|
14465
|
-
* - Gadget errors with full context (parameters, error message)
|
|
14466
|
-
* - Separate logging for LLM and gadget failures
|
|
14467
|
-
*
|
|
14468
|
-
* **Use cases:**
|
|
14469
|
-
* - Troubleshooting production issues
|
|
14470
|
-
* - Understanding error patterns and frequency
|
|
14471
|
-
* - Debugging error recovery behavior
|
|
14472
|
-
* - Collecting error metrics for monitoring
|
|
14473
|
-
*
|
|
14474
|
-
* **Performance:** Minimal overhead. Only logs when errors occur.
|
|
14475
|
-
*
|
|
14476
|
-
* @returns Hook configuration that can be passed to .withHooks()
|
|
14477
|
-
*
|
|
14478
|
-
* @example
|
|
14479
|
-
* ```typescript
|
|
14480
|
-
* // Basic error logging
|
|
14481
|
-
* await LLMist.createAgent()
|
|
14482
|
-
* .withHooks(HookPresets.errorLogging())
|
|
14483
|
-
* .withGadgets(Database)
|
|
14484
|
-
* .ask("Fetch user data");
|
|
14485
|
-
* // Output (on LLM error): ❌ LLM Error (iteration 1): Rate limit exceeded
|
|
14486
|
-
* // Model: gpt-5-nano
|
|
14487
|
-
* // Recovered: true
|
|
14488
|
-
* // Output (on gadget error): ❌ Gadget Error: Database
|
|
14489
|
-
* // Error: Connection timeout
|
|
14490
|
-
* // Parameters: {...}
|
|
14491
|
-
* ```
|
|
14492
|
-
*
|
|
14493
|
-
* @example
|
|
14494
|
-
* ```typescript
|
|
14495
|
-
* // Combine with monitoring for full context
|
|
14496
|
-
* .withHooks(HookPresets.merge(
|
|
14497
|
-
* HookPresets.monitoring(), // Includes errorLogging
|
|
14498
|
-
* customErrorAnalytics
|
|
14499
|
-
* ))
|
|
14500
|
-
* ```
|
|
14501
|
-
*
|
|
14502
|
-
* @example
|
|
14503
|
-
* ```typescript
|
|
14504
|
-
* // Error analytics collection
|
|
14505
|
-
* const errors: any[] = [];
|
|
14506
|
-
* .withHooks(HookPresets.merge(
|
|
14507
|
-
* HookPresets.errorLogging(),
|
|
14508
|
-
* {
|
|
14509
|
-
* observers: {
|
|
14510
|
-
* onLLMCallError: async (ctx) => {
|
|
14511
|
-
* errors.push({ type: 'llm', error: ctx.error, recovered: ctx.recovered });
|
|
14512
|
-
* },
|
|
14513
|
-
* },
|
|
14514
|
-
* }
|
|
14515
|
-
* ))
|
|
14516
|
-
* ```
|
|
14517
|
-
*
|
|
14518
|
-
* @see {@link https://github.com/zbigniewsobiecki/llmist/blob/main/docs/HOOKS.md#hookpresetserrorlogging | Full documentation}
|
|
14519
|
-
*/
|
|
14520
|
-
static errorLogging() {
|
|
14521
|
-
return {
|
|
14522
|
-
observers: {
|
|
14523
|
-
onLLMCallError: async (ctx) => {
|
|
14524
|
-
console.error(`\u274C LLM Error (iteration ${ctx.iteration}):`, ctx.error.message);
|
|
14525
|
-
console.error(` Model: ${ctx.options.model}`);
|
|
14526
|
-
console.error(` Recovered: ${ctx.recovered}`);
|
|
14527
|
-
},
|
|
14528
|
-
onGadgetExecutionComplete: async (ctx) => {
|
|
14529
|
-
if (ctx.error) {
|
|
14530
|
-
console.error(`\u274C Gadget Error: ${ctx.gadgetName}`);
|
|
14531
|
-
console.error(` Error: ${ctx.error}`);
|
|
14532
|
-
console.error(` Parameters:`, JSON.stringify(ctx.parameters, null, 2));
|
|
14533
|
-
}
|
|
14534
|
-
}
|
|
14535
|
-
}
|
|
14536
|
-
};
|
|
14537
|
-
}
|
|
14538
|
-
/**
|
|
14539
|
-
* Tracks context compaction events.
|
|
14540
|
-
*
|
|
14541
|
-
* **Output:**
|
|
14542
|
-
* - Compaction events with 🗜️ emoji
|
|
14543
|
-
* - Strategy name, tokens before/after, and savings
|
|
14544
|
-
* - Cumulative statistics
|
|
14545
|
-
*
|
|
14546
|
-
* **Use cases:**
|
|
14547
|
-
* - Monitoring long-running conversations
|
|
14548
|
-
* - Understanding when and how compaction occurs
|
|
14549
|
-
* - Debugging context management issues
|
|
14550
|
-
*
|
|
14551
|
-
* **Performance:** Minimal overhead. Simple console output.
|
|
14552
|
-
*
|
|
14553
|
-
* @returns Hook configuration that can be passed to .withHooks()
|
|
14554
|
-
*
|
|
14555
|
-
* @example
|
|
14556
|
-
* ```typescript
|
|
14557
|
-
* await LLMist.createAgent()
|
|
14558
|
-
* .withHooks(HookPresets.compactionTracking())
|
|
14559
|
-
* .ask("Your prompt");
|
|
14560
|
-
* ```
|
|
14561
|
-
*/
|
|
14562
|
-
static compactionTracking() {
|
|
14563
|
-
return {
|
|
14564
|
-
observers: {
|
|
14565
|
-
onCompaction: async (ctx) => {
|
|
14566
|
-
const saved = ctx.event.tokensBefore - ctx.event.tokensAfter;
|
|
14567
|
-
const percent = (saved / ctx.event.tokensBefore * 100).toFixed(1);
|
|
14568
|
-
console.log(
|
|
14569
|
-
`\u{1F5DC}\uFE0F Compaction (${ctx.event.strategy}): ${ctx.event.tokensBefore} \u2192 ${ctx.event.tokensAfter} tokens (saved ${saved}, ${percent}%)`
|
|
14570
|
-
);
|
|
14571
|
-
console.log(` Messages: ${ctx.event.messagesBefore} \u2192 ${ctx.event.messagesAfter}`);
|
|
14572
|
-
if (ctx.stats.totalCompactions > 1) {
|
|
14573
|
-
console.log(
|
|
14574
|
-
` Cumulative: ${ctx.stats.totalCompactions} compactions, ${ctx.stats.totalTokensSaved} tokens saved`
|
|
14575
|
-
);
|
|
14576
|
-
}
|
|
14577
|
-
}
|
|
14578
|
-
}
|
|
14579
|
-
};
|
|
14580
|
-
}
|
|
14581
|
-
/**
|
|
14582
|
-
* Returns empty hook configuration for clean output without any logging.
|
|
14583
|
-
*
|
|
14584
|
-
* **Output:**
|
|
14585
|
-
* - None. Returns {} (empty object).
|
|
14586
|
-
*
|
|
14587
|
-
* **Use cases:**
|
|
14588
|
-
* - Clean test output without console noise
|
|
14589
|
-
* - Production environments where logging is handled externally
|
|
14590
|
-
* - Baseline for custom hook development
|
|
14591
|
-
* - Temporary disable of all hook output
|
|
14592
|
-
*
|
|
14593
|
-
* **Performance:** Zero overhead. No-op hook configuration.
|
|
14594
|
-
*
|
|
14595
|
-
* @returns Empty hook configuration
|
|
14596
|
-
*
|
|
14597
|
-
* @example
|
|
14598
|
-
* ```typescript
|
|
14599
|
-
* // Clean test output
|
|
14600
|
-
* describe('Agent tests', () => {
|
|
14601
|
-
* it('should calculate correctly', async () => {
|
|
14602
|
-
* const result = await LLMist.createAgent()
|
|
14603
|
-
* .withHooks(HookPresets.silent()) // No console output
|
|
14604
|
-
* .withGadgets(Calculator)
|
|
14605
|
-
* .askAndCollect("What is 15 times 23?");
|
|
14606
|
-
*
|
|
14607
|
-
* expect(result).toContain("345");
|
|
14608
|
-
* });
|
|
14609
|
-
* });
|
|
14610
|
-
* ```
|
|
14611
|
-
*
|
|
14612
|
-
* @example
|
|
14613
|
-
* ```typescript
|
|
14614
|
-
* // Conditional silence based on environment
|
|
14615
|
-
* const isTesting = process.env.NODE_ENV === 'test';
|
|
14616
|
-
* .withHooks(isTesting ? HookPresets.silent() : HookPresets.monitoring())
|
|
14617
|
-
* ```
|
|
14618
|
-
*
|
|
14619
|
-
* @see {@link https://github.com/zbigniewsobiecki/llmist/blob/main/docs/HOOKS.md#hookpresetssilent | Full documentation}
|
|
14620
|
-
*/
|
|
14621
|
-
static silent() {
|
|
14622
|
-
return {};
|
|
14623
|
-
}
|
|
14624
|
-
/**
|
|
14625
|
-
* Combines multiple hook configurations into one.
|
|
14626
|
-
*
|
|
14627
|
-
* Merge allows you to compose preset and custom hooks for modular monitoring
|
|
14628
|
-
* configurations. Understanding merge behavior is crucial for proper composition.
|
|
14629
|
-
*
|
|
14630
|
-
* **Merge behavior:**
|
|
14631
|
-
* - **Observers:** Composed - all handlers run sequentially in order
|
|
14632
|
-
* - **Interceptors:** Last one wins - only the last interceptor applies
|
|
14633
|
-
* - **Controllers:** Last one wins - only the last controller applies
|
|
14634
|
-
*
|
|
14635
|
-
* **Why interceptors/controllers don't compose:**
|
|
14636
|
-
* - Interceptors have different signatures per method, making composition impractical
|
|
14637
|
-
* - Controllers return specific actions that can't be meaningfully combined
|
|
14638
|
-
* - Only observers support composition because they're read-only and independent
|
|
14639
|
-
*
|
|
14640
|
-
* **Use cases:**
|
|
14641
|
-
* - Combining multiple presets (logging + timing + tokens)
|
|
14642
|
-
* - Adding custom hooks to presets
|
|
14643
|
-
* - Building modular, reusable monitoring configurations
|
|
14644
|
-
* - Environment-specific hook composition
|
|
14645
|
-
*
|
|
14646
|
-
* **Performance:** Minimal overhead for merging. Runtime performance depends on merged hooks.
|
|
14647
|
-
*
|
|
14648
|
-
* @param hookSets - Variable number of hook configurations to merge
|
|
14649
|
-
* @returns Single merged hook configuration with composed/overridden handlers
|
|
14650
|
-
*
|
|
14651
|
-
* @example
|
|
14652
|
-
* ```typescript
|
|
14653
|
-
* // Combine multiple presets
|
|
14654
|
-
* .withHooks(HookPresets.merge(
|
|
14655
|
-
* HookPresets.logging(),
|
|
14656
|
-
* HookPresets.timing(),
|
|
14657
|
-
* HookPresets.tokenTracking()
|
|
14658
|
-
* ))
|
|
14659
|
-
* // All observers from all three presets will run
|
|
14660
|
-
* ```
|
|
14661
|
-
*
|
|
14662
|
-
* @example
|
|
14663
|
-
* ```typescript
|
|
14664
|
-
* // Add custom observer to preset (both run)
|
|
14665
|
-
* .withHooks(HookPresets.merge(
|
|
14666
|
-
* HookPresets.timing(),
|
|
14667
|
-
* {
|
|
14668
|
-
* observers: {
|
|
14669
|
-
* onLLMCallComplete: async (ctx) => {
|
|
14670
|
-
* await saveMetrics({ tokens: ctx.usage?.totalTokens });
|
|
14671
|
-
* },
|
|
14672
|
-
* },
|
|
14673
|
-
* }
|
|
14674
|
-
* ))
|
|
14675
|
-
* ```
|
|
14676
|
-
*
|
|
14677
|
-
* @example
|
|
14678
|
-
* ```typescript
|
|
14679
|
-
* // Multiple interceptors (last wins!)
|
|
14680
|
-
* .withHooks(HookPresets.merge(
|
|
14681
|
-
* {
|
|
14682
|
-
* interceptors: {
|
|
14683
|
-
* interceptTextChunk: (chunk) => chunk.toUpperCase(), // Ignored
|
|
14684
|
-
* },
|
|
14685
|
-
* },
|
|
14686
|
-
* {
|
|
14687
|
-
* interceptors: {
|
|
14688
|
-
* interceptTextChunk: (chunk) => chunk.toLowerCase(), // This wins
|
|
14689
|
-
* },
|
|
14690
|
-
* }
|
|
14691
|
-
* ))
|
|
14692
|
-
* // Result: text will be lowercase
|
|
14693
|
-
* ```
|
|
14694
|
-
*
|
|
14695
|
-
* @example
|
|
14696
|
-
* ```typescript
|
|
14697
|
-
* // Modular environment-based configuration
|
|
14698
|
-
* const baseHooks = HookPresets.errorLogging();
|
|
14699
|
-
* const devHooks = HookPresets.merge(baseHooks, HookPresets.monitoring({ verbose: true }));
|
|
14700
|
-
* const prodHooks = HookPresets.merge(baseHooks, HookPresets.tokenTracking());
|
|
14701
|
-
*
|
|
14702
|
-
* const hooks = process.env.NODE_ENV === 'production' ? prodHooks : devHooks;
|
|
14703
|
-
* .withHooks(hooks)
|
|
14704
|
-
* ```
|
|
14705
|
-
*
|
|
14706
|
-
* @see {@link https://github.com/zbigniewsobiecki/llmist/blob/main/docs/HOOKS.md#hookpresetsmergehooksets | Full documentation}
|
|
14707
|
-
*/
|
|
14708
|
-
static merge(...hookSets) {
|
|
14709
|
-
const merged = {
|
|
14710
|
-
observers: {},
|
|
14711
|
-
interceptors: {},
|
|
14712
|
-
controllers: {}
|
|
14713
|
-
};
|
|
14714
|
-
for (const hooks of hookSets) {
|
|
14715
|
-
if (hooks.observers) {
|
|
14716
|
-
for (const [key, handler] of Object.entries(hooks.observers)) {
|
|
14717
|
-
const typedKey = key;
|
|
14718
|
-
if (merged.observers[typedKey]) {
|
|
14719
|
-
const existing = merged.observers[typedKey];
|
|
14720
|
-
merged.observers[typedKey] = async (ctx) => {
|
|
14721
|
-
await existing(ctx);
|
|
14722
|
-
await handler(ctx);
|
|
14723
|
-
};
|
|
14724
|
-
} else {
|
|
14725
|
-
merged.observers[typedKey] = handler;
|
|
14726
|
-
}
|
|
14727
|
-
}
|
|
14728
|
-
}
|
|
14729
|
-
if (hooks.interceptors) {
|
|
14730
|
-
Object.assign(merged.interceptors, hooks.interceptors);
|
|
14731
|
-
}
|
|
14732
|
-
if (hooks.controllers) {
|
|
14733
|
-
Object.assign(merged.controllers, hooks.controllers);
|
|
14734
|
-
}
|
|
14735
|
-
}
|
|
14736
|
-
return merged;
|
|
14737
|
-
}
|
|
14738
|
-
/**
|
|
14739
|
-
* Composite preset combining logging, timing, tokenTracking, and errorLogging.
|
|
14740
|
-
*
|
|
14741
|
-
* This is the recommended preset for development and initial production deployments,
|
|
14742
|
-
* providing comprehensive observability with a single method call.
|
|
14743
|
-
*
|
|
14744
|
-
* **Includes:**
|
|
14745
|
-
* - All output from `logging()` preset (with optional verbosity)
|
|
14746
|
-
* - All output from `timing()` preset (execution times)
|
|
14747
|
-
* - All output from `tokenTracking()` preset (token usage)
|
|
14748
|
-
* - All output from `errorLogging()` preset (error details)
|
|
14749
|
-
*
|
|
14750
|
-
* **Output format:**
|
|
14751
|
-
* - Event logging: [LLM]/[GADGET] messages
|
|
14752
|
-
* - Timing: ⏱️ emoji with milliseconds
|
|
14753
|
-
* - Tokens: 📊 emoji with per-call and cumulative counts
|
|
14754
|
-
* - Errors: ❌ emoji with full error details
|
|
14755
|
-
*
|
|
14756
|
-
* **Use cases:**
|
|
14757
|
-
* - Full observability during development
|
|
14758
|
-
* - Comprehensive monitoring in production
|
|
14759
|
-
* - One-liner for complete agent visibility
|
|
14760
|
-
* - Troubleshooting and debugging with full context
|
|
14761
|
-
*
|
|
14762
|
-
* **Performance:** Combined overhead of all four presets, but still minimal in practice.
|
|
14763
|
-
*
|
|
14764
|
-
* @param options - Monitoring options
|
|
14765
|
-
* @param options.verbose - Passed to logging() preset for detailed output. Default: false
|
|
14766
|
-
* @returns Merged hook configuration combining all monitoring presets
|
|
14767
|
-
*
|
|
14768
|
-
* @example
|
|
14769
|
-
* ```typescript
|
|
14770
|
-
* // Basic monitoring (recommended for development)
|
|
14771
|
-
* await LLMist.createAgent()
|
|
14772
|
-
* .withHooks(HookPresets.monitoring())
|
|
14773
|
-
* .withGadgets(Calculator, Weather)
|
|
14774
|
-
* .ask("What is 15 times 23, and what's the weather in NYC?");
|
|
14775
|
-
* // Output: All events, timing, tokens, and errors in one place
|
|
14776
|
-
* ```
|
|
14777
|
-
*
|
|
14778
|
-
* @example
|
|
14779
|
-
* ```typescript
|
|
14780
|
-
* // Verbose monitoring with full details
|
|
14781
|
-
* await LLMist.createAgent()
|
|
14782
|
-
* .withHooks(HookPresets.monitoring({ verbose: true }))
|
|
14783
|
-
* .ask("Your prompt");
|
|
14784
|
-
* // Output includes: parameters, results, and complete responses
|
|
14785
|
-
* ```
|
|
14786
|
-
*
|
|
14787
|
-
* @example
|
|
14788
|
-
* ```typescript
|
|
14789
|
-
* // Environment-based monitoring
|
|
14790
|
-
* const isDev = process.env.NODE_ENV === 'development';
|
|
14791
|
-
* .withHooks(HookPresets.monitoring({ verbose: isDev }))
|
|
14792
|
-
* ```
|
|
14793
|
-
*
|
|
14794
|
-
* @see {@link https://github.com/zbigniewsobiecki/llmist/blob/main/docs/HOOKS.md#hookpresetsmonitoringoptions | Full documentation}
|
|
14795
|
-
*/
|
|
14796
|
-
static monitoring(options = {}) {
|
|
14797
|
-
return _HookPresets.merge(
|
|
14798
|
-
_HookPresets.logging(options),
|
|
14799
|
-
_HookPresets.timing(),
|
|
14800
|
-
_HookPresets.tokenTracking(),
|
|
14801
|
-
_HookPresets.errorLogging()
|
|
14802
|
-
);
|
|
14803
|
-
}
|
|
14804
|
-
};
|
|
15006
|
+
init_file_logging();
|
|
15007
|
+
init_hook_presets();
|
|
14805
15008
|
|
|
14806
15009
|
// src/agent/compaction/index.ts
|
|
14807
15010
|
init_config();
|
|
@@ -14814,6 +15017,7 @@ init_gadget_output_store();
|
|
|
14814
15017
|
|
|
14815
15018
|
// src/agent/hints.ts
|
|
14816
15019
|
init_prompt_config();
|
|
15020
|
+
init_hook_presets();
|
|
14817
15021
|
function iterationProgressHint(options) {
|
|
14818
15022
|
const { timing: timing2 = "always", showUrgency = true, template } = options ?? {};
|
|
14819
15023
|
return {
|
|
@@ -15603,9 +15807,11 @@ function getHostExports2(ctx) {
|
|
|
15603
15807
|
filterRootEvents,
|
|
15604
15808
|
format,
|
|
15605
15809
|
formatBytes,
|
|
15810
|
+
formatCallNumber,
|
|
15606
15811
|
formatDate,
|
|
15607
15812
|
formatDuration,
|
|
15608
15813
|
formatLLMError,
|
|
15814
|
+
formatLlmRequest,
|
|
15609
15815
|
gadgetError,
|
|
15610
15816
|
gadgetSuccess,
|
|
15611
15817
|
getErrorMessage,
|