@granular-software/sdk 0.4.37 → 0.4.38

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. package/dist/agent-evals.d.mts +16 -3
  2. package/dist/agent-evals.d.ts +16 -3
  3. package/dist/agent-evals.js +341 -105
  4. package/dist/agent-evals.js.map +1 -1
  5. package/dist/agent-evals.mjs +340 -105
  6. package/dist/agent-evals.mjs.map +1 -1
  7. package/dist/cli/index.js +129 -3
  8. package/dist/client-BNbWA9jQ.d.ts +1064 -0
  9. package/dist/client-HDJgcJC5.d.mts +1064 -0
  10. package/dist/index.d.mts +3 -2
  11. package/dist/index.d.ts +3 -2
  12. package/dist/index.js +231 -3
  13. package/dist/index.js.map +1 -1
  14. package/dist/index.mjs +225 -4
  15. package/dist/index.mjs.map +1 -1
  16. package/dist/spend-rzS1rlFr.d.mts +1559 -0
  17. package/dist/spend-rzS1rlFr.d.ts +1559 -0
  18. package/dist/spend.d.mts +2 -0
  19. package/dist/spend.d.ts +2 -0
  20. package/dist/spend.js +111 -0
  21. package/dist/spend.js.map +1 -0
  22. package/dist/spend.mjs +107 -0
  23. package/dist/spend.mjs.map +1 -0
  24. package/package.json +7 -1
  25. package/dist/client-eE9nTfvp.d.mts +0 -2483
  26. package/dist/client-eE9nTfvp.d.ts +0 -2483
package/dist/agent-evals.d.mts CHANGED
@@ -1,8 +1,9 @@
1
- import { P as Prompt, f as EnvironmentSession, c as SessionHeapSnapshot, br as ManifestContent, aR as RecordObjectOptions, T as ToolWithHandler, G as Granular, C as ConnectOptions, J as CreateEnvironmentData, k as GranularOptions } from './client-eE9nTfvp.mjs';
1
+ import { a as EnvironmentSession, G as Granular } from './client-HDJgcJC5.mjs';
2
+ import { G as GranularSpendContext, h as OpenAITokenSpend, P as Prompt, c as SessionHeapSnapshot, bH as ManifestContent, b5 as RecordObjectOptions, T as ToolWithHandler, C as ConnectOptions, a6 as CreateEnvironmentData, q as GranularOptions } from './spend-rzS1rlFr.mjs';
2
3
  import { GranularAgentToolInfo, GeneratedJobCodeIssue, HarnessControllerBudgets } from './agent-harness.mjs';
3
- import '@granular-software/policy-engine';
4
4
  import '@automerge/automerge';
5
5
  import '@automerge/automerge/slim';
6
+ import '@granular-software/policy-engine';
6
7
 
7
8
  type EvalMatcher = string | RegExp;
8
9
  interface AgentEvalPromptInteraction {
@@ -23,6 +24,7 @@ interface AgentEvalGenerationInput {
23
24
  tools?: GranularAgentToolInfo[];
24
25
  repairIssues?: GeneratedJobCodeIssue[];
25
26
  onTextDelta?: (delta: string) => void | Promise<void>;
27
+ usageContext?: GranularSpendContext;
26
28
  }
27
29
  interface AgentEvalGenerationOutput {
28
30
  reply?: string;
@@ -51,6 +53,15 @@ interface AgentEvalTokenUsage {
51
53
  totalCostUsd: number;
52
54
  missingUsageCalls: number;
53
55
  }
56
+ interface AgentEvalOpenAIUsageEvent extends OpenAITokenSpend {
57
+ spendEventId?: string;
58
+ source: "openai";
59
+ lineItemType: "llm_tokens";
60
+ operation: "chat.completions";
61
+ requestId?: string | null;
62
+ usageContext?: GranularSpendContext;
63
+ rawUsage: unknown;
64
+ }
54
65
  type AgentEvalTurnGenerator = (input: AgentEvalGenerationInput) => Promise<AgentEvalGenerationOutput>;
55
66
  interface ScriptedPromptRule {
56
67
  type?: Prompt["type"];
@@ -278,6 +289,8 @@ declare function createOpenAIChatTurnGenerator(options: {
278
289
  baseUrl?: string;
279
290
  temperature?: number;
280
291
  headers?: Record<string, string>;
292
+ usageContext?: GranularSpendContext;
293
+ onUsage?: (usage: AgentEvalOpenAIUsageEvent) => void | Promise<void>;
281
294
  }): AgentEvalTurnGenerator;
282
295
  declare function runAgentEvalSuite(options: {
283
296
  harness: ReturnType<typeof createAgentEvalHarness>;
@@ -379,4 +392,4 @@ declare const createHumanResponder: typeof createScriptedPromptResponder;
379
392
  declare const createOpenAIGenerator: typeof createOpenAIChatTurnGenerator;
380
393
  declare const createTestArtifactsDirectory: typeof createTimestampedArtifactDirectory;
381
394
 
382
- export { type AgentEvalCheck, type AgentEvalCheckContext, type AgentEvalCompletedTurn, type AgentEvalConversation, type AgentEvalExpectations, type AgentEvalGenerationAttempt, type AgentEvalGenerationInput, type AgentEvalGenerationOutput, type AgentEvalHarnessOptions, type AgentEvalInspection, type AgentEvalPendingTurn, type AgentEvalPrepareContext, type AgentEvalPromptInteraction, type AgentEvalPromptResponder, type AgentEvalResult, type AgentEvalScenario, type AgentEvalSetup, type AgentEvalSetupContext, type AgentEvalStep, type AgentEvalStepResult, type AgentEvalSuiteResult, type AgentEvalTokenUsage, type AgentEvalTurnGenerator, type AgentTestTarget, type AgentTesterOptions, type EvalMatcher, type ScriptedPromptRule, createAgentEvalHarness, createAgentTester, createHumanResponder, createOpenAIChatTurnGenerator, createOpenAIGenerator, createScriptedPromptResponder, createTestArtifactsDirectory, createTimestampedArtifactDirectory, runAgentEvalSuite, runAgentTests };
395
+ export { type AgentEvalCheck, type AgentEvalCheckContext, type AgentEvalCompletedTurn, type AgentEvalConversation, type AgentEvalExpectations, type AgentEvalGenerationAttempt, type AgentEvalGenerationInput, type AgentEvalGenerationOutput, type AgentEvalHarnessOptions, type AgentEvalInspection, type AgentEvalOpenAIUsageEvent, type AgentEvalPendingTurn, type AgentEvalPrepareContext, type AgentEvalPromptInteraction, type AgentEvalPromptResponder, type AgentEvalResult, type AgentEvalScenario, type AgentEvalSetup, type AgentEvalSetupContext, type AgentEvalStep, type AgentEvalStepResult, type AgentEvalSuiteResult, type AgentEvalTokenUsage, type AgentEvalTurnGenerator, type AgentTestTarget, type AgentTesterOptions, type EvalMatcher, type ScriptedPromptRule, createAgentEvalHarness, createAgentTester, createHumanResponder, createOpenAIChatTurnGenerator, createOpenAIGenerator, createScriptedPromptResponder, createTestArtifactsDirectory, createTimestampedArtifactDirectory, runAgentEvalSuite, runAgentTests };
package/dist/agent-evals.d.ts CHANGED
@@ -1,8 +1,9 @@
1
- import { P as Prompt, f as EnvironmentSession, c as SessionHeapSnapshot, br as ManifestContent, aR as RecordObjectOptions, T as ToolWithHandler, G as Granular, C as ConnectOptions, J as CreateEnvironmentData, k as GranularOptions } from './client-eE9nTfvp.js';
1
+ import { a as EnvironmentSession, G as Granular } from './client-BNbWA9jQ.js';
2
+ import { G as GranularSpendContext, h as OpenAITokenSpend, P as Prompt, c as SessionHeapSnapshot, bH as ManifestContent, b5 as RecordObjectOptions, T as ToolWithHandler, C as ConnectOptions, a6 as CreateEnvironmentData, q as GranularOptions } from './spend-rzS1rlFr.js';
2
3
  import { GranularAgentToolInfo, GeneratedJobCodeIssue, HarnessControllerBudgets } from './agent-harness.js';
3
- import '@granular-software/policy-engine';
4
4
  import '@automerge/automerge';
5
5
  import '@automerge/automerge/slim';
6
+ import '@granular-software/policy-engine';
6
7
 
7
8
  type EvalMatcher = string | RegExp;
8
9
  interface AgentEvalPromptInteraction {
@@ -23,6 +24,7 @@ interface AgentEvalGenerationInput {
23
24
  tools?: GranularAgentToolInfo[];
24
25
  repairIssues?: GeneratedJobCodeIssue[];
25
26
  onTextDelta?: (delta: string) => void | Promise<void>;
27
+ usageContext?: GranularSpendContext;
26
28
  }
27
29
  interface AgentEvalGenerationOutput {
28
30
  reply?: string;
@@ -51,6 +53,15 @@ interface AgentEvalTokenUsage {
51
53
  totalCostUsd: number;
52
54
  missingUsageCalls: number;
53
55
  }
56
+ interface AgentEvalOpenAIUsageEvent extends OpenAITokenSpend {
57
+ spendEventId?: string;
58
+ source: "openai";
59
+ lineItemType: "llm_tokens";
60
+ operation: "chat.completions";
61
+ requestId?: string | null;
62
+ usageContext?: GranularSpendContext;
63
+ rawUsage: unknown;
64
+ }
54
65
  type AgentEvalTurnGenerator = (input: AgentEvalGenerationInput) => Promise<AgentEvalGenerationOutput>;
55
66
  interface ScriptedPromptRule {
56
67
  type?: Prompt["type"];
@@ -278,6 +289,8 @@ declare function createOpenAIChatTurnGenerator(options: {
278
289
  baseUrl?: string;
279
290
  temperature?: number;
280
291
  headers?: Record<string, string>;
292
+ usageContext?: GranularSpendContext;
293
+ onUsage?: (usage: AgentEvalOpenAIUsageEvent) => void | Promise<void>;
281
294
  }): AgentEvalTurnGenerator;
282
295
  declare function runAgentEvalSuite(options: {
283
296
  harness: ReturnType<typeof createAgentEvalHarness>;
@@ -379,4 +392,4 @@ declare const createHumanResponder: typeof createScriptedPromptResponder;
379
392
  declare const createOpenAIGenerator: typeof createOpenAIChatTurnGenerator;
380
393
  declare const createTestArtifactsDirectory: typeof createTimestampedArtifactDirectory;
381
394
 
382
- export { type AgentEvalCheck, type AgentEvalCheckContext, type AgentEvalCompletedTurn, type AgentEvalConversation, type AgentEvalExpectations, type AgentEvalGenerationAttempt, type AgentEvalGenerationInput, type AgentEvalGenerationOutput, type AgentEvalHarnessOptions, type AgentEvalInspection, type AgentEvalPendingTurn, type AgentEvalPrepareContext, type AgentEvalPromptInteraction, type AgentEvalPromptResponder, type AgentEvalResult, type AgentEvalScenario, type AgentEvalSetup, type AgentEvalSetupContext, type AgentEvalStep, type AgentEvalStepResult, type AgentEvalSuiteResult, type AgentEvalTokenUsage, type AgentEvalTurnGenerator, type AgentTestTarget, type AgentTesterOptions, type EvalMatcher, type ScriptedPromptRule, createAgentEvalHarness, createAgentTester, createHumanResponder, createOpenAIChatTurnGenerator, createOpenAIGenerator, createScriptedPromptResponder, createTestArtifactsDirectory, createTimestampedArtifactDirectory, runAgentEvalSuite, runAgentTests };
395
+ export { type AgentEvalCheck, type AgentEvalCheckContext, type AgentEvalCompletedTurn, type AgentEvalConversation, type AgentEvalExpectations, type AgentEvalGenerationAttempt, type AgentEvalGenerationInput, type AgentEvalGenerationOutput, type AgentEvalHarnessOptions, type AgentEvalInspection, type AgentEvalOpenAIUsageEvent, type AgentEvalPendingTurn, type AgentEvalPrepareContext, type AgentEvalPromptInteraction, type AgentEvalPromptResponder, type AgentEvalResult, type AgentEvalScenario, type AgentEvalSetup, type AgentEvalSetupContext, type AgentEvalStep, type AgentEvalStepResult, type AgentEvalSuiteResult, type AgentEvalTokenUsage, type AgentEvalTurnGenerator, type AgentTestTarget, type AgentTesterOptions, type EvalMatcher, type ScriptedPromptRule, createAgentEvalHarness, createAgentTester, createHumanResponder, createOpenAIChatTurnGenerator, createOpenAIGenerator, createScriptedPromptResponder, createTestArtifactsDirectory, createTimestampedArtifactDirectory, runAgentEvalSuite, runAgentTests };