@granular-software/sdk 0.4.36 → 0.4.38

package/README.md

@@ -70,7 +70,7 @@ const env = await granular.openEnvironment({
   ontology: "my-ontology",
   tag: "dev",
   userId: "user_123",
-  permissions: ["agent"],
+  permissions: ["allow-all"],
   name: "Jane Doe", // optional
   email: "jane@example.com", // optional
 });
@@ -268,7 +268,7 @@ const env = await granular.openEnvironment({
   ontology: "library-app",
   tag: "dev",
   userId: "user_123",
-  permissions: ["agent"],
+  permissions: ["allow-all"],
 });
 
 const [tolkien, lotr] = await env.recordObjects(
@@ -380,6 +380,86 @@ await granular.ontology(env.sandboxId).effects.registerMany([
 ]);
 ```
 
+## Permission Profiles and Policies
+
+Policies are versioned with the ontology. Author role profiles as JSON files under `permissions/`, then run `granular build` or `granular deploy`; the CLI validates and syncs those files before the build.
+
+Built-in profiles are materialized by `granular init` and `granular pull`:
+
+- `allow-all`: exposes every declared action unless manifest logic denies it.
+- `confirm-all`: exposes every declared action but requires confirmation.
+- `deny-all`: exposes no executable actions.
+
+Manifest-level logic policies live on effects and apply to every profile:
+
+```json
+{
+  "withEffect": {
+    "name": "approve_ticket",
+    "attachedClass": "ticket",
+    "inputSchema": { "type": "object", "properties": { "note": { "type": "string" } } },
+    "outputSchema": { "type": "object", "properties": { "ok": { "type": "boolean" } } },
+    "metamodels": {
+      "policies": {
+        "denyWhen": [
+          {
+            "reason": "Locked tickets cannot be approved",
+            "when": { "object": { "path": "locked", "operator": "eq", "booleanValue": true } }
+          }
+        ]
+      }
+    }
+  }
+}
+```
+
+Permission profile files define role-specific access:
+
+```json
+{
+  "schemaVersion": 1,
+  "name": "reviewer",
+  "defaults": { "actionPolicy": "deny" },
+  "policies": [
+    {
+      "action": "approve_ticket",
+      "on": "ticket",
+      "decision": "allow",
+      "reason": "Active low-risk tickets can be approved",
+      "when": {
+        "all": [
+          { "object": { "path": "risk_score", "operator": "lte", "numberValue": 50 } },
+          { "stateMachine": { "machine": "lifecycle", "operator": "eq", "stringValue": "active" } }
+        ]
+      }
+    }
+  ],
+  "actions": [
+    { "action": "comment_ticket", "on": "ticket", "allow": "always" }
+  ]
+}
+```
+
+Supported profile features:
+
+- `defaults.actionPolicy`: `allow`, `confirm`, or `deny` (omitted means `deny`).
+- `extends`: inherit one parent profile; children cannot define `defaults`.
+- `policies[]`: cross-action or targeted rules with `decision` / `outcome` and `when`.
+- `actions[]`: `allow`, `confirm`, `deny`, `allowWhen`, `confirmWhen`, `denyWhen`, and numeric `limits`.
+- Conditions can read `input`, target `object` fields, and target `stateMachine` state, with `all`, `any`, and `not`.
+
+Decision precedence is deterministic: manifest deny, profile deny, confirmation rules, profile allow, then the profile default. Runtime blocks denied actions before the effect provider is invoked; confirmation rules use the normal human prompt flow.
+
+Useful CLI checks:
+
+```bash
+granular permissions validate
+granular permissions preview --profile reviewer --action approve_ticket --on ticket \
+  --input '{"note":"ok"}' \
+  --object '{"risk_score":40,"locked":false}' \
+  --state-machines '{"lifecycle":"active"}'
+```
+
 ## Domain Synthesis (Auto-Generated Types)
 
 After applying the manifest and publishing tools, the sandbox gets **auto-generated TypeScript classes** in `./sandbox-tools`:

package/dist/agent-evals.d.mts

@@ -1,7 +1,9 @@
-import { P as Prompt, f as EnvironmentSession, c as SessionHeapSnapshot, bl as ManifestContent, aO as RecordObjectOptions, T as ToolWithHandler, G as Granular, C as ConnectOptions, J as CreateEnvironmentData, k as GranularOptions } from './client-Cq8onk2D.mjs';
-import { GeneratedJobCodeIssue, HarnessControllerBudgets } from './agent-harness.mjs';
+import { a as EnvironmentSession, G as Granular } from './client-HDJgcJC5.mjs';
+import { G as GranularSpendContext, h as OpenAITokenSpend, P as Prompt, c as SessionHeapSnapshot, bH as ManifestContent, b5 as RecordObjectOptions, T as ToolWithHandler, C as ConnectOptions, a6 as CreateEnvironmentData, q as GranularOptions } from './spend-rzS1rlFr.mjs';
+import { GranularAgentToolInfo, GeneratedJobCodeIssue, HarnessControllerBudgets } from './agent-harness.mjs';
 import '@automerge/automerge';
 import '@automerge/automerge/slim';
+import '@granular-software/policy-engine';
 
 type EvalMatcher = string | RegExp;
 interface AgentEvalPromptInteraction {
@@ -19,12 +21,46 @@ interface AgentEvalGenerationInput {
     }>;
     request: string;
     attempt: number;
+    tools?: GranularAgentToolInfo[];
     repairIssues?: GeneratedJobCodeIssue[];
+    onTextDelta?: (delta: string) => void | Promise<void>;
+    usageContext?: GranularSpendContext;
 }
 interface AgentEvalGenerationOutput {
     reply?: string;
     code?: string;
     raw?: unknown;
+    generationAttempts?: AgentEvalGenerationAttempt[];
+}
+interface AgentEvalGenerationAttempt {
+    attempt: number;
+    request: string;
+    repairIssues?: GeneratedJobCodeIssue[];
+    reply?: string;
+    code?: string;
+    raw?: unknown;
+}
+interface AgentEvalTokenUsage {
+    calls: number;
+    inputTokens: number;
+    cachedInputTokens: number;
+    uncachedInputTokens: number;
+    outputTokens: number;
+    totalTokens: number;
+    inputCostUsd: number;
+    cachedInputCostUsd: number;
+    outputCostUsd: number;
+    totalCostUsd: number;
+    missingUsageCalls: number;
+}
+interface AgentEvalOpenAIUsageEvent extends OpenAITokenSpend {
+    spendEventId?: string;
+    source: "openai";
+    lineItemType: "llm_tokens";
+    operation: "chat.completions";
+    requestId?: string | null;
+    usageContext?: GranularSpendContext;
+    rawUsage: unknown;
 }
 type AgentEvalTurnGenerator = (input: AgentEvalGenerationInput) => Promise<AgentEvalGenerationOutput>;
 interface ScriptedPromptRule {
@@ -101,6 +137,8 @@ interface AgentEvalStep {
 }
 interface AgentEvalScenario {
     id: string;
+    description?: string;
+    jobSource?: "llm" | "hardcoded" | "mixed";
     request?: string;
     steps?: AgentEvalStep[];
     setup?: AgentEvalSetup;
@@ -134,6 +172,7 @@ interface AgentEvalResult {
     actionSummary: string[];
     promptInteractions: AgentEvalPromptInteraction[];
     verification: unknown | null;
+    tokenUsage?: AgentEvalTokenUsage;
     steps?: AgentEvalStepResult[];
     turnDir: string;
     error?: string;
@@ -164,6 +203,39 @@ interface AgentEvalConversation {
     promptEvents: PendingPromptEvent[];
     artifactDir: string;
     turnCount: number;
+    logTurns: AgentEvalSessionLogTurn[];
+}
+interface AgentEvalSessionLogIteration {
+    iteration: number;
+    request: string;
+    systemPrompt: string;
+    generationReply?: string;
+    generatedCode?: string;
+    rawGeneration?: unknown;
+    generationAttempts?: AgentEvalGenerationAttempt[];
+    tokenUsage?: AgentEvalTokenUsage;
+    responseText?: string;
+    terminalKind?: "reply" | "closure";
+    actionSummary?: string[];
+    promptInteractions?: AgentEvalPromptInteraction[];
+    continuation?: unknown;
+    result?: unknown;
+    error?: string;
+}
+interface AgentEvalSessionLogTurn {
+    turnNumber: number;
+    turnId: string;
+    request: string;
+    turnDir: string;
+    iterations: AgentEvalSessionLogIteration[];
+    completed?: {
+        responseText: string;
+        terminalKind: "reply" | "closure";
+        actionSummary: string[];
+        promptInteractions: AgentEvalPromptInteraction[];
+        result?: unknown;
+    };
+    error?: string;
 }
 interface AgentEvalPendingTurn {
     conversation: AgentEvalConversation;
@@ -217,6 +289,8 @@ declare function createOpenAIChatTurnGenerator(options: {
     baseUrl?: string;
     temperature?: number;
     headers?: Record<string, string>;
+    usageContext?: GranularSpendContext;
+    onUsage?: (usage: AgentEvalOpenAIUsageEvent) => void | Promise<void>;
 }): AgentEvalTurnGenerator;
 declare function runAgentEvalSuite(options: {
     harness: ReturnType<typeof createAgentEvalHarness>;
@@ -318,4 +392,4 @@ declare const createHumanResponder: typeof createScriptedPromptResponder;
 declare const createOpenAIGenerator: typeof createOpenAIChatTurnGenerator;
 declare const createTestArtifactsDirectory: typeof createTimestampedArtifactDirectory;
 
-export { type AgentEvalCheck, type AgentEvalCheckContext, type AgentEvalCompletedTurn, type AgentEvalConversation, type AgentEvalExpectations, type AgentEvalGenerationInput, type AgentEvalGenerationOutput, type AgentEvalHarnessOptions, type AgentEvalInspection, type AgentEvalPendingTurn, type AgentEvalPrepareContext, type AgentEvalPromptInteraction, type AgentEvalPromptResponder, type AgentEvalResult, type AgentEvalScenario, type AgentEvalSetup, type AgentEvalSetupContext, type AgentEvalStep, type AgentEvalStepResult, type AgentEvalSuiteResult, type AgentEvalTurnGenerator, type AgentTestTarget, type AgentTesterOptions, type EvalMatcher, type ScriptedPromptRule, createAgentEvalHarness, createAgentTester, createHumanResponder, createOpenAIChatTurnGenerator, createOpenAIGenerator, createScriptedPromptResponder, createTestArtifactsDirectory, createTimestampedArtifactDirectory, runAgentEvalSuite, runAgentTests };
+export { type AgentEvalCheck, type AgentEvalCheckContext, type AgentEvalCompletedTurn, type AgentEvalConversation, type AgentEvalExpectations, type AgentEvalGenerationAttempt, type AgentEvalGenerationInput, type AgentEvalGenerationOutput, type AgentEvalHarnessOptions, type AgentEvalInspection, type AgentEvalOpenAIUsageEvent, type AgentEvalPendingTurn, type AgentEvalPrepareContext, type AgentEvalPromptInteraction, type AgentEvalPromptResponder, type AgentEvalResult, type AgentEvalScenario, type AgentEvalSetup, type AgentEvalSetupContext, type AgentEvalStep, type AgentEvalStepResult, type AgentEvalSuiteResult, type AgentEvalTokenUsage, type AgentEvalTurnGenerator, type AgentTestTarget, type AgentTesterOptions, type EvalMatcher, type ScriptedPromptRule, createAgentEvalHarness, createAgentTester, createHumanResponder, createOpenAIChatTurnGenerator, createOpenAIGenerator, createScriptedPromptResponder, createTestArtifactsDirectory, createTimestampedArtifactDirectory, runAgentEvalSuite, runAgentTests };

package/dist/agent-evals.d.ts

@@ -1,7 +1,9 @@
-import { P as Prompt, f as EnvironmentSession, c as SessionHeapSnapshot, bl as ManifestContent, aO as RecordObjectOptions, T as ToolWithHandler, G as Granular, C as ConnectOptions, J as CreateEnvironmentData, k as GranularOptions } from './client-Cq8onk2D.js';
-import { GeneratedJobCodeIssue, HarnessControllerBudgets } from './agent-harness.js';
+import { a as EnvironmentSession, G as Granular } from './client-BNbWA9jQ.js';
+import { G as GranularSpendContext, h as OpenAITokenSpend, P as Prompt, c as SessionHeapSnapshot, bH as ManifestContent, b5 as RecordObjectOptions, T as ToolWithHandler, C as ConnectOptions, a6 as CreateEnvironmentData, q as GranularOptions } from './spend-rzS1rlFr.js';
+import { GranularAgentToolInfo, GeneratedJobCodeIssue, HarnessControllerBudgets } from './agent-harness.js';
 import '@automerge/automerge';
 import '@automerge/automerge/slim';
+import '@granular-software/policy-engine';
 
 type EvalMatcher = string | RegExp;
 interface AgentEvalPromptInteraction {
@@ -19,12 +21,46 @@ interface AgentEvalGenerationInput {
     }>;
     request: string;
     attempt: number;
+    tools?: GranularAgentToolInfo[];
     repairIssues?: GeneratedJobCodeIssue[];
+    onTextDelta?: (delta: string) => void | Promise<void>;
+    usageContext?: GranularSpendContext;
 }
 interface AgentEvalGenerationOutput {
     reply?: string;
     code?: string;
     raw?: unknown;
+    generationAttempts?: AgentEvalGenerationAttempt[];
+}
+interface AgentEvalGenerationAttempt {
+    attempt: number;
+    request: string;
+    repairIssues?: GeneratedJobCodeIssue[];
+    reply?: string;
+    code?: string;
+    raw?: unknown;
+}
+interface AgentEvalTokenUsage {
+    calls: number;
+    inputTokens: number;
+    cachedInputTokens: number;
+    uncachedInputTokens: number;
+    outputTokens: number;
+    totalTokens: number;
+    inputCostUsd: number;
+    cachedInputCostUsd: number;
+    outputCostUsd: number;
+    totalCostUsd: number;
+    missingUsageCalls: number;
+}
+interface AgentEvalOpenAIUsageEvent extends OpenAITokenSpend {
+    spendEventId?: string;
+    source: "openai";
+    lineItemType: "llm_tokens";
+    operation: "chat.completions";
+    requestId?: string | null;
+    usageContext?: GranularSpendContext;
+    rawUsage: unknown;
 }
 type AgentEvalTurnGenerator = (input: AgentEvalGenerationInput) => Promise<AgentEvalGenerationOutput>;
 interface ScriptedPromptRule {
@@ -101,6 +137,8 @@ interface AgentEvalStep {
 }
 interface AgentEvalScenario {
     id: string;
+    description?: string;
+    jobSource?: "llm" | "hardcoded" | "mixed";
     request?: string;
     steps?: AgentEvalStep[];
     setup?: AgentEvalSetup;
@@ -134,6 +172,7 @@ interface AgentEvalResult {
     actionSummary: string[];
     promptInteractions: AgentEvalPromptInteraction[];
     verification: unknown | null;
+    tokenUsage?: AgentEvalTokenUsage;
     steps?: AgentEvalStepResult[];
     turnDir: string;
     error?: string;
@@ -164,6 +203,39 @@ interface AgentEvalConversation {
     promptEvents: PendingPromptEvent[];
     artifactDir: string;
     turnCount: number;
+    logTurns: AgentEvalSessionLogTurn[];
+}
+interface AgentEvalSessionLogIteration {
+    iteration: number;
+    request: string;
+    systemPrompt: string;
+    generationReply?: string;
+    generatedCode?: string;
+    rawGeneration?: unknown;
+    generationAttempts?: AgentEvalGenerationAttempt[];
+    tokenUsage?: AgentEvalTokenUsage;
+    responseText?: string;
+    terminalKind?: "reply" | "closure";
+    actionSummary?: string[];
+    promptInteractions?: AgentEvalPromptInteraction[];
+    continuation?: unknown;
+    result?: unknown;
+    error?: string;
+}
+interface AgentEvalSessionLogTurn {
+    turnNumber: number;
+    turnId: string;
+    request: string;
+    turnDir: string;
+    iterations: AgentEvalSessionLogIteration[];
+    completed?: {
+        responseText: string;
+        terminalKind: "reply" | "closure";
+        actionSummary: string[];
+        promptInteractions: AgentEvalPromptInteraction[];
+        result?: unknown;
+    };
+    error?: string;
 }
 interface AgentEvalPendingTurn {
     conversation: AgentEvalConversation;
@@ -217,6 +289,8 @@ declare function createOpenAIChatTurnGenerator(options: {
     baseUrl?: string;
     temperature?: number;
     headers?: Record<string, string>;
+    usageContext?: GranularSpendContext;
+    onUsage?: (usage: AgentEvalOpenAIUsageEvent) => void | Promise<void>;
 }): AgentEvalTurnGenerator;
 declare function runAgentEvalSuite(options: {
     harness: ReturnType<typeof createAgentEvalHarness>;
@@ -318,4 +392,4 @@ declare const createHumanResponder: typeof createScriptedPromptResponder;
 declare const createOpenAIGenerator: typeof createOpenAIChatTurnGenerator;
 declare const createTestArtifactsDirectory: typeof createTimestampedArtifactDirectory;
 
-export { type AgentEvalCheck, type AgentEvalCheckContext, type AgentEvalCompletedTurn, type AgentEvalConversation, type AgentEvalExpectations, type AgentEvalGenerationInput, type AgentEvalGenerationOutput, type AgentEvalHarnessOptions, type AgentEvalInspection, type AgentEvalPendingTurn, type AgentEvalPrepareContext, type AgentEvalPromptInteraction, type AgentEvalPromptResponder, type AgentEvalResult, type AgentEvalScenario, type AgentEvalSetup, type AgentEvalSetupContext, type AgentEvalStep, type AgentEvalStepResult, type AgentEvalSuiteResult, type AgentEvalTurnGenerator, type AgentTestTarget, type AgentTesterOptions, type EvalMatcher, type ScriptedPromptRule, createAgentEvalHarness, createAgentTester, createHumanResponder, createOpenAIChatTurnGenerator, createOpenAIGenerator, createScriptedPromptResponder, createTestArtifactsDirectory, createTimestampedArtifactDirectory, runAgentEvalSuite, runAgentTests };
+export { type AgentEvalCheck, type AgentEvalCheckContext, type AgentEvalCompletedTurn, type AgentEvalConversation, type AgentEvalExpectations, type AgentEvalGenerationAttempt, type AgentEvalGenerationInput, type AgentEvalGenerationOutput, type AgentEvalHarnessOptions, type AgentEvalInspection, type AgentEvalOpenAIUsageEvent, type AgentEvalPendingTurn, type AgentEvalPrepareContext, type AgentEvalPromptInteraction, type AgentEvalPromptResponder, type AgentEvalResult, type AgentEvalScenario, type AgentEvalSetup, type AgentEvalSetupContext, type AgentEvalStep, type AgentEvalStepResult, type AgentEvalSuiteResult, type AgentEvalTokenUsage, type AgentEvalTurnGenerator, type AgentTestTarget, type AgentTesterOptions, type EvalMatcher, type ScriptedPromptRule, createAgentEvalHarness, createAgentTester, createHumanResponder, createOpenAIChatTurnGenerator, createOpenAIGenerator, createScriptedPromptResponder, createTestArtifactsDirectory, createTimestampedArtifactDirectory, runAgentEvalSuite, runAgentTests };