@granular-software/sdk 0.4.37 → 0.4.39

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/dist/agent-evals.d.mts +17 -3
  2. package/dist/agent-evals.d.ts +17 -3
  3. package/dist/agent-evals.js +1067 -153
  4. package/dist/agent-evals.js.map +1 -1
  5. package/dist/agent-evals.mjs +1066 -154
  6. package/dist/agent-evals.mjs.map +1 -1
  7. package/dist/agent-harness.d.mts +9 -2
  8. package/dist/agent-harness.d.ts +9 -2
  9. package/dist/agent-harness.js +273 -5
  10. package/dist/agent-harness.js.map +1 -1
  11. package/dist/agent-harness.mjs +271 -6
  12. package/dist/agent-harness.mjs.map +1 -1
  13. package/dist/cli/index.js +293 -34
  14. package/dist/client-BZ8NuQ_e.d.ts +1080 -0
  15. package/dist/client-CBQFvuKf.d.mts +1080 -0
  16. package/dist/index.d.mts +4 -3
  17. package/dist/index.d.ts +4 -3
  18. package/dist/index.js +668 -39
  19. package/dist/index.js.map +1 -1
  20. package/dist/index.mjs +659 -40
  21. package/dist/index.mjs.map +1 -1
  22. package/dist/spend-tAz2a16I.d.mts +1593 -0
  23. package/dist/spend-tAz2a16I.d.ts +1593 -0
  24. package/dist/spend.d.mts +2 -0
  25. package/dist/spend.d.ts +2 -0
  26. package/dist/spend.js +111 -0
  27. package/dist/spend.js.map +1 -0
  28. package/dist/spend.mjs +107 -0
  29. package/dist/spend.mjs.map +1 -0
  30. package/package.json +7 -1
  31. package/dist/client-eE9nTfvp.d.mts +0 -2483
  32. package/dist/client-eE9nTfvp.d.ts +0 -2483
package/dist/agent-evals.d.mts CHANGED
@@ -1,8 +1,9 @@
1
- import { P as Prompt, f as EnvironmentSession, c as SessionHeapSnapshot, br as ManifestContent, aR as RecordObjectOptions, T as ToolWithHandler, G as Granular, C as ConnectOptions, J as CreateEnvironmentData, k as GranularOptions } from './client-eE9nTfvp.mjs';
1
+ import { a as EnvironmentSession, G as Granular } from './client-CBQFvuKf.mjs';
2
+ import { G as GranularSpendContext, h as OpenAITokenSpend, P as Prompt, c as SessionHeapSnapshot, bM as ManifestContent, ba as RecordObjectOptions, T as ToolWithHandler, C as ConnectOptions, a6 as CreateEnvironmentData, q as GranularOptions } from './spend-tAz2a16I.mjs';
2
3
  import { GranularAgentToolInfo, GeneratedJobCodeIssue, HarnessControllerBudgets } from './agent-harness.mjs';
3
- import '@granular-software/policy-engine';
4
4
  import '@automerge/automerge';
5
5
  import '@automerge/automerge/slim';
6
+ import '@granular-software/policy-engine';
6
7
 
7
8
  type EvalMatcher = string | RegExp;
8
9
  interface AgentEvalPromptInteraction {
@@ -23,6 +24,7 @@ interface AgentEvalGenerationInput {
23
24
  tools?: GranularAgentToolInfo[];
24
25
  repairIssues?: GeneratedJobCodeIssue[];
25
26
  onTextDelta?: (delta: string) => void | Promise<void>;
27
+ usageContext?: GranularSpendContext;
26
28
  }
27
29
  interface AgentEvalGenerationOutput {
28
30
  reply?: string;
@@ -51,6 +53,15 @@ interface AgentEvalTokenUsage {
51
53
  totalCostUsd: number;
52
54
  missingUsageCalls: number;
53
55
  }
56
+ interface AgentEvalOpenAIUsageEvent extends OpenAITokenSpend {
57
+ spendEventId?: string;
58
+ source: "openai";
59
+ lineItemType: "llm_tokens";
60
+ operation: "chat.completions";
61
+ requestId?: string | null;
62
+ usageContext?: GranularSpendContext;
63
+ rawUsage: unknown;
64
+ }
54
65
  type AgentEvalTurnGenerator = (input: AgentEvalGenerationInput) => Promise<AgentEvalGenerationOutput>;
55
66
  interface ScriptedPromptRule {
56
67
  type?: Prompt["type"];
@@ -278,7 +289,10 @@ declare function createOpenAIChatTurnGenerator(options: {
278
289
  baseUrl?: string;
279
290
  temperature?: number;
280
291
  headers?: Record<string, string>;
292
+ usageContext?: GranularSpendContext;
293
+ onUsage?: (usage: AgentEvalOpenAIUsageEvent) => void | Promise<void>;
281
294
  }): AgentEvalTurnGenerator;
295
+ declare function generateTurnWithRepair(generator: AgentEvalTurnGenerator, input: AgentEvalGenerationInput): Promise<AgentEvalGenerationOutput>;
282
296
  declare function runAgentEvalSuite(options: {
283
297
  harness: ReturnType<typeof createAgentEvalHarness>;
284
298
  scenarios: AgentEvalScenario[];
@@ -379,4 +393,4 @@ declare const createHumanResponder: typeof createScriptedPromptResponder;
379
393
  declare const createOpenAIGenerator: typeof createOpenAIChatTurnGenerator;
380
394
  declare const createTestArtifactsDirectory: typeof createTimestampedArtifactDirectory;
381
395
 
382
- export { type AgentEvalCheck, type AgentEvalCheckContext, type AgentEvalCompletedTurn, type AgentEvalConversation, type AgentEvalExpectations, type AgentEvalGenerationAttempt, type AgentEvalGenerationInput, type AgentEvalGenerationOutput, type AgentEvalHarnessOptions, type AgentEvalInspection, type AgentEvalPendingTurn, type AgentEvalPrepareContext, type AgentEvalPromptInteraction, type AgentEvalPromptResponder, type AgentEvalResult, type AgentEvalScenario, type AgentEvalSetup, type AgentEvalSetupContext, type AgentEvalStep, type AgentEvalStepResult, type AgentEvalSuiteResult, type AgentEvalTokenUsage, type AgentEvalTurnGenerator, type AgentTestTarget, type AgentTesterOptions, type EvalMatcher, type ScriptedPromptRule, createAgentEvalHarness, createAgentTester, createHumanResponder, createOpenAIChatTurnGenerator, createOpenAIGenerator, createScriptedPromptResponder, createTestArtifactsDirectory, createTimestampedArtifactDirectory, runAgentEvalSuite, runAgentTests };
396
+ export { type AgentEvalCheck, type AgentEvalCheckContext, type AgentEvalCompletedTurn, type AgentEvalConversation, type AgentEvalExpectations, type AgentEvalGenerationAttempt, type AgentEvalGenerationInput, type AgentEvalGenerationOutput, type AgentEvalHarnessOptions, type AgentEvalInspection, type AgentEvalOpenAIUsageEvent, type AgentEvalPendingTurn, type AgentEvalPrepareContext, type AgentEvalPromptInteraction, type AgentEvalPromptResponder, type AgentEvalResult, type AgentEvalScenario, type AgentEvalSetup, type AgentEvalSetupContext, type AgentEvalStep, type AgentEvalStepResult, type AgentEvalSuiteResult, type AgentEvalTokenUsage, type AgentEvalTurnGenerator, type AgentTestTarget, type AgentTesterOptions, type EvalMatcher, type ScriptedPromptRule, createAgentEvalHarness, createAgentTester, createHumanResponder, createOpenAIChatTurnGenerator, createOpenAIGenerator, createScriptedPromptResponder, createTestArtifactsDirectory, createTimestampedArtifactDirectory, generateTurnWithRepair, runAgentEvalSuite, runAgentTests };
package/dist/agent-evals.d.ts CHANGED
@@ -1,8 +1,9 @@
1
- import { P as Prompt, f as EnvironmentSession, c as SessionHeapSnapshot, br as ManifestContent, aR as RecordObjectOptions, T as ToolWithHandler, G as Granular, C as ConnectOptions, J as CreateEnvironmentData, k as GranularOptions } from './client-eE9nTfvp.js';
1
+ import { a as EnvironmentSession, G as Granular } from './client-BZ8NuQ_e.js';
2
+ import { G as GranularSpendContext, h as OpenAITokenSpend, P as Prompt, c as SessionHeapSnapshot, bM as ManifestContent, ba as RecordObjectOptions, T as ToolWithHandler, C as ConnectOptions, a6 as CreateEnvironmentData, q as GranularOptions } from './spend-tAz2a16I.js';
2
3
  import { GranularAgentToolInfo, GeneratedJobCodeIssue, HarnessControllerBudgets } from './agent-harness.js';
3
- import '@granular-software/policy-engine';
4
4
  import '@automerge/automerge';
5
5
  import '@automerge/automerge/slim';
6
+ import '@granular-software/policy-engine';
6
7
 
7
8
  type EvalMatcher = string | RegExp;
8
9
  interface AgentEvalPromptInteraction {
@@ -23,6 +24,7 @@ interface AgentEvalGenerationInput {
23
24
  tools?: GranularAgentToolInfo[];
24
25
  repairIssues?: GeneratedJobCodeIssue[];
25
26
  onTextDelta?: (delta: string) => void | Promise<void>;
27
+ usageContext?: GranularSpendContext;
26
28
  }
27
29
  interface AgentEvalGenerationOutput {
28
30
  reply?: string;
@@ -51,6 +53,15 @@ interface AgentEvalTokenUsage {
51
53
  totalCostUsd: number;
52
54
  missingUsageCalls: number;
53
55
  }
56
+ interface AgentEvalOpenAIUsageEvent extends OpenAITokenSpend {
57
+ spendEventId?: string;
58
+ source: "openai";
59
+ lineItemType: "llm_tokens";
60
+ operation: "chat.completions";
61
+ requestId?: string | null;
62
+ usageContext?: GranularSpendContext;
63
+ rawUsage: unknown;
64
+ }
54
65
  type AgentEvalTurnGenerator = (input: AgentEvalGenerationInput) => Promise<AgentEvalGenerationOutput>;
55
66
  interface ScriptedPromptRule {
56
67
  type?: Prompt["type"];
@@ -278,7 +289,10 @@ declare function createOpenAIChatTurnGenerator(options: {
278
289
  baseUrl?: string;
279
290
  temperature?: number;
280
291
  headers?: Record<string, string>;
292
+ usageContext?: GranularSpendContext;
293
+ onUsage?: (usage: AgentEvalOpenAIUsageEvent) => void | Promise<void>;
281
294
  }): AgentEvalTurnGenerator;
295
+ declare function generateTurnWithRepair(generator: AgentEvalTurnGenerator, input: AgentEvalGenerationInput): Promise<AgentEvalGenerationOutput>;
282
296
  declare function runAgentEvalSuite(options: {
283
297
  harness: ReturnType<typeof createAgentEvalHarness>;
284
298
  scenarios: AgentEvalScenario[];
@@ -379,4 +393,4 @@ declare const createHumanResponder: typeof createScriptedPromptResponder;
379
393
  declare const createOpenAIGenerator: typeof createOpenAIChatTurnGenerator;
380
394
  declare const createTestArtifactsDirectory: typeof createTimestampedArtifactDirectory;
381
395
 
382
- export { type AgentEvalCheck, type AgentEvalCheckContext, type AgentEvalCompletedTurn, type AgentEvalConversation, type AgentEvalExpectations, type AgentEvalGenerationAttempt, type AgentEvalGenerationInput, type AgentEvalGenerationOutput, type AgentEvalHarnessOptions, type AgentEvalInspection, type AgentEvalPendingTurn, type AgentEvalPrepareContext, type AgentEvalPromptInteraction, type AgentEvalPromptResponder, type AgentEvalResult, type AgentEvalScenario, type AgentEvalSetup, type AgentEvalSetupContext, type AgentEvalStep, type AgentEvalStepResult, type AgentEvalSuiteResult, type AgentEvalTokenUsage, type AgentEvalTurnGenerator, type AgentTestTarget, type AgentTesterOptions, type EvalMatcher, type ScriptedPromptRule, createAgentEvalHarness, createAgentTester, createHumanResponder, createOpenAIChatTurnGenerator, createOpenAIGenerator, createScriptedPromptResponder, createTestArtifactsDirectory, createTimestampedArtifactDirectory, runAgentEvalSuite, runAgentTests };
396
+ export { type AgentEvalCheck, type AgentEvalCheckContext, type AgentEvalCompletedTurn, type AgentEvalConversation, type AgentEvalExpectations, type AgentEvalGenerationAttempt, type AgentEvalGenerationInput, type AgentEvalGenerationOutput, type AgentEvalHarnessOptions, type AgentEvalInspection, type AgentEvalOpenAIUsageEvent, type AgentEvalPendingTurn, type AgentEvalPrepareContext, type AgentEvalPromptInteraction, type AgentEvalPromptResponder, type AgentEvalResult, type AgentEvalScenario, type AgentEvalSetup, type AgentEvalSetupContext, type AgentEvalStep, type AgentEvalStepResult, type AgentEvalSuiteResult, type AgentEvalTokenUsage, type AgentEvalTurnGenerator, type AgentTestTarget, type AgentTesterOptions, type EvalMatcher, type ScriptedPromptRule, createAgentEvalHarness, createAgentTester, createHumanResponder, createOpenAIChatTurnGenerator, createOpenAIGenerator, createScriptedPromptResponder, createTestArtifactsDirectory, createTimestampedArtifactDirectory, generateTurnWithRepair, runAgentEvalSuite, runAgentTests };