@vellumai/assistant 0.5.14 → 0.5.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +2 -2
- package/docs/architecture/integrations.md +15 -14
- package/knip.json +3 -1
- package/openapi.yaml +11 -43
- package/package.json +1 -1
- package/src/__tests__/assistant-feature-flags-integration.test.ts +3 -375
- package/src/__tests__/ces-rpc-credential-backend.test.ts +4 -1
- package/src/__tests__/checker.test.ts +59 -0
- package/src/__tests__/cli-command-risk-guard.test.ts +98 -10
- package/src/__tests__/cli-memory.test.ts +372 -0
- package/src/__tests__/computer-use-skill-manifest-regression.test.ts +12 -2
- package/src/__tests__/config-schema.test.ts +0 -2
- package/src/__tests__/config-watcher-feature-flags.test.ts +211 -0
- package/src/__tests__/conversation-runtime-assembly.test.ts +7 -4
- package/src/__tests__/conversation-slash-commands.test.ts +2 -6
- package/src/__tests__/conversation-usage.test.ts +1 -0
- package/src/__tests__/credential-security-e2e.test.ts +4 -1
- package/src/__tests__/docker-signing-key-bootstrap.test.ts +7 -73
- package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +6 -7
- package/src/__tests__/guardian-routing-invariants.test.ts +151 -0
- package/src/__tests__/heartbeat-service.test.ts +1 -3
- package/src/__tests__/intent-routing.test.ts +6 -18
- package/src/__tests__/log-export-workspace.test.ts +2 -28
- package/src/__tests__/managed-skill-lifecycle.test.ts +7 -37
- package/src/__tests__/managed-store.test.ts +2 -10
- package/src/__tests__/messaging-send-tool.test.ts +6 -6
- package/src/__tests__/migration-cross-version-compatibility.test.ts +1 -29
- package/src/__tests__/migration-export-http.test.ts +3 -34
- package/src/__tests__/migration-import-commit-http.test.ts +1 -29
- package/src/__tests__/migration-import-preflight-http.test.ts +3 -34
- package/src/__tests__/no-domain-routing-in-prompt-guard.test.ts +2 -1
- package/src/__tests__/oauth-apps-routes.test.ts +120 -10
- package/src/__tests__/oauth-connect-orchestrator.test.ts +709 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +2 -1
- package/src/__tests__/oauth-provider-visibility.test.ts +149 -0
- package/src/__tests__/oauth-providers-routes.test.ts +5 -2
- package/src/__tests__/oauth-store.test.ts +0 -5
- package/src/__tests__/outlook-messaging-provider.test.ts +576 -0
- package/src/__tests__/path-policy.test.ts +2 -17
- package/src/__tests__/permission-types.test.ts +0 -1
- package/src/__tests__/platform-callback-registration.test.ts +3 -7
- package/src/__tests__/provider-commit-message-generator.test.ts +0 -1
- package/src/__tests__/provider-error-scenarios.test.ts +0 -2
- package/src/__tests__/qdrant-manager.test.ts +68 -21
- package/src/__tests__/require-fresh-approval.test.ts +0 -1
- package/src/__tests__/sandbox-diagnostics.test.ts +20 -29
- package/src/__tests__/scaffold-managed-skill-tool.test.ts +2 -10
- package/src/__tests__/secret-allowlist.test.ts +20 -35
- package/src/__tests__/shell-credential-ref.test.ts +0 -5
- package/src/__tests__/skill-load-feature-flag.test.ts +2 -43
- package/src/__tests__/skill-load-inline-command.test.ts +3 -65
- package/src/__tests__/skill-load-inline-includes.test.ts +3 -65
- package/src/__tests__/skill-load-tool.test.ts +3 -67
- package/src/__tests__/skill-memory.test.ts +362 -119
- package/src/__tests__/skills.test.ts +22 -49
- package/src/__tests__/slack-channel-config.test.ts +2 -21
- package/src/__tests__/starter-bundle.test.ts +2 -8
- package/src/__tests__/stt-hints.test.ts +7 -2
- package/src/__tests__/system-prompt.test.ts +25 -45
- package/src/__tests__/task-compiler.test.ts +0 -21
- package/src/__tests__/task-management-tools.test.ts +0 -21
- package/src/__tests__/task-memory-cleanup.test.ts +0 -21
- package/src/__tests__/task-runner.test.ts +0 -21
- package/src/__tests__/task-scheduler.test.ts +0 -21
- package/src/__tests__/terminal-tools.test.ts +1 -17
- package/src/__tests__/token-estimator-accuracy.benchmark.test.ts +0 -79
- package/src/__tests__/tool-approval-handler.test.ts +1 -20
- package/src/__tests__/tool-execution-abort-cleanup.test.ts +2 -11
- package/src/__tests__/tool-execution-pipeline.benchmark.test.ts +1 -25
- package/src/__tests__/tool-executor-lifecycle-events.test.ts +0 -1
- package/src/__tests__/tool-executor.test.ts +0 -1
- package/src/__tests__/tool-grant-request-escalation.test.ts +1 -20
- package/src/__tests__/tool-preview-lifecycle.test.ts +0 -20
- package/src/__tests__/trust-store.test.ts +9 -41
- package/src/__tests__/trusted-contact-approval-notifier.test.ts +1 -30
- package/src/__tests__/trusted-contact-inline-approval-integration.test.ts +1 -21
- package/src/__tests__/trusted-contact-lifecycle-notifications.test.ts +0 -22
- package/src/__tests__/trusted-contact-multichannel.test.ts +0 -22
- package/src/__tests__/trusted-contact-verification.test.ts +0 -22
- package/src/__tests__/turn-boundary-resolution.test.ts +0 -28
- package/src/__tests__/twilio-provider.test.ts +0 -16
- package/src/__tests__/twilio-routes-twiml.test.ts +7 -12
- package/src/__tests__/twilio-routes.test.ts +0 -24
- package/src/__tests__/update-bulletin.test.ts +17 -89
- package/src/__tests__/usage-cache-backfill-migration.test.ts +0 -20
- package/src/__tests__/usage-routes.test.ts +0 -21
- package/src/__tests__/user-reference.test.ts +1 -5
- package/src/__tests__/vbundle-pax-and-symlink.test.ts +4 -34
- package/src/__tests__/vellum-self-knowledge-inline-command.test.ts +2 -53
- package/src/__tests__/voice-invite-redemption.test.ts +0 -21
- package/src/__tests__/voice-scoped-grant-consumer.test.ts +0 -24
- package/src/__tests__/voice-session-bridge.test.ts +0 -21
- package/src/__tests__/workspace-migration-009-backfill-conversation-disk-view.test.ts +2 -23
- package/src/__tests__/workspace-migration-012-rename-conversation-disk-view-dirs.test.ts +2 -2
- package/src/__tests__/workspace-migration-013-repair-conversation-disk-view.test.ts +2 -23
- package/src/__tests__/workspace-migration-down-functions.test.ts +0 -6
- package/src/acp/client-handler.ts +1 -2
- package/src/cli/__tests__/notifications.test.ts +0 -22
- package/src/cli/cli-memory.ts +176 -0
- package/src/cli/commands/oauth/__tests__/providers-update.test.ts +1 -1
- package/src/cli/commands/oauth/connect.ts +15 -0
- package/src/cli/commands/oauth/providers.ts +49 -42
- package/src/cli/commands/platform/__tests__/connect.test.ts +2 -48
- package/src/cli/commands/platform/__tests__/disconnect.test.ts +2 -48
- package/src/cli/commands/platform/__tests__/status.test.ts +0 -50
- package/src/config/bundled-skills/computer-use/TOOLS.json +7 -7
- package/src/config/bundled-skills/messaging/SKILL.md +17 -2
- package/src/config/bundled-skills/settings/TOOLS.json +3 -3
- package/src/config/feature-flag-registry.json +16 -0
- package/src/config/loader.ts +4 -0
- package/src/config/schemas/security.ts +0 -6
- package/src/config/schemas/services.ts +8 -0
- package/src/context/window-manager.ts +28 -9
- package/src/credential-execution/approval-bridge.ts +0 -1
- package/src/daemon/config-watcher.ts +51 -0
- package/src/daemon/conversation-agent-loop.ts +3 -2
- package/src/daemon/conversation-process.ts +1 -0
- package/src/daemon/conversation-usage.ts +1 -0
- package/src/daemon/handlers/skills.ts +9 -1
- package/src/daemon/lifecycle.ts +13 -4
- package/src/daemon/message-types/conversations.ts +1 -0
- package/src/daemon/providers-setup.ts +2 -0
- package/src/daemon/server.ts +26 -22
- package/src/events/domain-events.ts +1 -2
- package/src/memory/db-init.ts +9 -0
- package/src/memory/job-handlers/batch-extraction.ts +16 -4
- package/src/memory/job-handlers/embedding.test.ts +3 -27
- package/src/memory/job-handlers/journal-carry-forward.test.ts +1 -29
- package/src/memory/llm-usage-store.ts +35 -2
- package/src/memory/migrations/201-oauth-providers-feature-flag.ts +11 -0
- package/src/memory/migrations/202-drop-callback-transport-column.ts +13 -0
- package/src/memory/migrations/index.ts +2 -0
- package/src/memory/qdrant-manager.ts +26 -5
- package/src/memory/query-expansion.ts +1 -1
- package/src/memory/retriever.test.ts +22 -20
- package/src/memory/retriever.ts +10 -2
- package/src/memory/schema/oauth.ts +1 -1
- package/src/memory/search/mmr.ts +8 -5
- package/src/memory/slack-thread-store.ts +17 -0
- package/src/messaging/providers/outlook/adapter.ts +193 -0
- package/src/messaging/providers/outlook/client.ts +311 -0
- package/src/messaging/providers/outlook/types.ts +83 -0
- package/src/notifications/adapters/slack.ts +1 -1
- package/src/oauth/__tests__/identity-verifier.test.ts +1 -1
- package/src/oauth/connect-orchestrator.ts +10 -3
- package/src/oauth/oauth-store.ts +10 -11
- package/src/oauth/provider-serializer.ts +3 -0
- package/src/oauth/provider-visibility.ts +16 -0
- package/src/oauth/seed-providers.ts +49 -17
- package/src/permissions/checker.ts +39 -7
- package/src/permissions/types.ts +2 -4
- package/src/prompts/journal-context.ts +9 -11
- package/src/prompts/system-prompt.ts +3 -64
- package/src/prompts/templates/UPDATES.md +6 -0
- package/src/runtime/auth/__tests__/credential-service.test.ts +1 -27
- package/src/runtime/auth/__tests__/token-service.test.ts +1 -25
- package/src/runtime/auth/route-policy.ts +0 -4
- package/src/runtime/guardian-reply-router.ts +6 -2
- package/src/runtime/routes/conversation-query-routes.ts +2 -58
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +43 -2
- package/src/runtime/routes/memory-item-routes.test.ts +0 -17
- package/src/runtime/routes/memory-item-routes.ts +103 -12
- package/src/runtime/routes/oauth-apps.ts +18 -1
- package/src/runtime/routes/oauth-providers.ts +13 -1
- package/src/runtime/routes/settings-routes.ts +1 -0
- package/src/runtime/routes/usage-routes.ts +19 -2
- package/src/runtime/routes/work-items-routes.test.ts +0 -21
- package/src/runtime/routes/workspace-routes.test.ts +3 -27
- package/src/security/secret-allowlist.ts +4 -4
- package/src/skills/skill-memory.ts +62 -23
- package/src/tools/memory/handlers.test.ts +1 -29
- package/src/tools/permission-checker.ts +0 -18
- package/src/tools/skills/skill-script-runner.ts +1 -1
- package/src/util/device-id.ts +3 -65
- package/src/workspace/git-service.ts +27 -6
package/ARCHITECTURE.md
CHANGED
|
@@ -637,7 +637,7 @@ The assistant feature-flag resolver (`src/config/assistant-feature-flags.ts`) is
|
|
|
637
637
|
| Enforcement Point | Module | Effect |
|
|
638
638
|
| ---------------------------------- | ------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
639
639
|
| **1. Client skill list** | `resolveSkillStates()` in `config/skill-state.ts` | Skills with flag OFF are excluded from the resolved list returned to clients (macOS skill list, settings UI). The skill never appears in the client. |
|
|
640
|
-
| **2.
|
|
640
|
+
| **2. Capability memory seeding** | `seedCatalogSkillMemories()` in `skills/skill-memory.ts` | Skills with flag OFF are excluded from capability memory seeding. The model cannot discover them via semantic recall. |
|
|
641
641
|
| **3. `skill_load` tool** | `executeSkillLoad()` in `tools/skills/load.ts` | If the model attempts to load a flagged-off skill by name, the tool returns an error: `"skill is currently unavailable (disabled by feature flag)"`. |
|
|
642
642
|
| **4. Runtime tool projection** | `projectSkillTools()` in `daemon/conversation-skill-tools.ts` | Even if a skill was previously active in a session (has `<loaded_skill>` markers in history), the per-turn projection drops it when the flag is OFF. Already-registered tools are unregistered. |
|
|
643
643
|
| **5. Included child skills** | `executeSkillLoad()` in `tools/skills/load.ts` | When a parent skill includes children via the `includes` directive, each child is independently checked against its feature flag. Flagged-off children are silently excluded from the loaded skill content. |
|
|
@@ -653,7 +653,7 @@ All six enforcement points derive the flag key via `skillFlagKey(skill)` — whi
|
|
|
653
653
|
| ----------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
654
654
|
| `src/config/assistant-feature-flags.ts` | Canonical resolver: `isAssistantFeatureFlagEnabled()`, `getAssistantFeatureFlagDefaults()`, registry loader |
|
|
655
655
|
| `src/config/skill-state.ts` | `skillFlagKey(skill)` — returns canonical flag key for skills with a `featureFlag` frontmatter field, `undefined` otherwise; `resolveSkillStates()` — enforcement point 1 |
|
|
656
|
-
| `src/
|
|
656
|
+
| `src/skills/skill-memory.ts` | `seedCatalogSkillMemories()` — enforcement point 2 |
|
|
657
657
|
| `src/tools/skills/load.ts` | `executeSkillLoad()` — enforcement points 3 and 5 |
|
|
658
658
|
| `src/daemon/conversation-skill-tools.ts` | `projectSkillTools()` — enforcement point 4 |
|
|
659
659
|
| `src/config/schema.ts` | `AssistantConfig` Zod schema definition (feature flag values are no longer stored here) |
|
|
@@ -164,18 +164,18 @@ sequenceDiagram
|
|
|
164
164
|
|
|
165
165
|
### Key Design Decisions
|
|
166
166
|
|
|
167
|
-
| Decision | Rationale
|
|
168
|
-
| ------------------------------------------ |
|
|
169
|
-
| PKCE by default, optional client_secret | Desktop apps prefer PKCE; some providers (Slack) require a secret, which is stored in the credential store (`oauth_app/{id}/client_secret`) for autonomous refresh
|
|
170
|
-
| Shared connect orchestrator | All OAuth providers route through `orchestrateOAuthConnect()`, which resolves profiles, enforces scope policy, runs the flow, stores tokens, and verifies identity. Adding a provider is a declarative profile entry, not new orchestration code
|
|
171
|
-
| Canonical credential naming | All reads and writes use `client_id`/`client_secret` as canonical field names
|
|
172
|
-
|
|
|
173
|
-
| Unified `MessagingProvider` interface | All platforms implement the same contract; generic tools work immediately for new providers
|
|
174
|
-
| Provider auto-selection | If only one provider is connected, tools skip the `platform` parameter — seamless single-platform UX
|
|
175
|
-
| Token expiry in SQLite oauth-store | `oauth_connections.expires_at` column tracks token expiry; `TokenManager` reads it for proactive refresh with 5min buffer. No separate metadata store needed
|
|
176
|
-
| Confidence scores on medium-risk tools | LLM self-reports confidence (0-1); enables future trust calibration without blocking execution
|
|
177
|
-
| Platform-specific extension tools | Operations unique to one platform (e.g. Gmail labels, Slack reactions) are separate tools, not forced into the generic interface
|
|
178
|
-
| Identity verification before token storage | OAuth2 tokens are only persisted after a successful identity verification call, preventing storage of invalid or mismatched credentials
|
|
167
|
+
| Decision | Rationale |
|
|
168
|
+
| ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
169
|
+
| PKCE by default, optional client_secret | Desktop apps prefer PKCE; some providers (Slack) require a secret, which is stored in the credential store (`oauth_app/{id}/client_secret`) for autonomous refresh |
|
|
170
|
+
| Shared connect orchestrator | All OAuth providers route through `orchestrateOAuthConnect()`, which resolves profiles, enforces scope policy, runs the flow, stores tokens, and verifies identity. Adding a provider is a declarative profile entry, not new orchestration code |
|
|
171
|
+
| Canonical credential naming | All reads and writes use `client_id`/`client_secret` as canonical field names |
|
|
172
|
+
| Caller-driven callback transport | Transport (`loopback` or `gateway`) is chosen per-flow via the `callbackTransport` option on the connect API, defaulting to loopback. Desktop clients use loopback (no tunnel needed); web clients can pass `callback_transport: "gateway"`. Provider configuration no longer dictates transport. |
|
|
173
|
+
| Unified `MessagingProvider` interface | All platforms implement the same contract; generic tools work immediately for new providers |
|
|
174
|
+
| Provider auto-selection | If only one provider is connected, tools skip the `platform` parameter — seamless single-platform UX |
|
|
175
|
+
| Token expiry in SQLite oauth-store | `oauth_connections.expires_at` column tracks token expiry; `TokenManager` reads it for proactive refresh with 5min buffer. No separate metadata store needed |
|
|
176
|
+
| Confidence scores on medium-risk tools | LLM self-reports confidence (0-1); enables future trust calibration without blocking execution |
|
|
177
|
+
| Platform-specific extension tools | Operations unique to one platform (e.g. Gmail labels, Slack reactions) are separate tools, not forced into the generic interface |
|
|
178
|
+
| Identity verification before token storage | OAuth2 tokens are only persisted after a successful identity verification call, preventing storage of invalid or mismatched credentials |
|
|
179
179
|
|
|
180
180
|
### Source Files
|
|
181
181
|
|
|
@@ -217,7 +217,7 @@ Each provider row includes:
|
|
|
217
217
|
|
|
218
218
|
| Column group | Fields | Purpose |
|
|
219
219
|
| ------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- |
|
|
220
|
-
| **Protocol** | `authUrl`, `tokenUrl`, `tokenEndpointAuthMethod`, `
|
|
220
|
+
| **Protocol** | `authUrl`, `tokenUrl`, `tokenEndpointAuthMethod`, `extraParams`, `loopbackPort` | OAuth2 flow parameters |
|
|
221
221
|
| **Scopes** | `defaultScopes`, `scopePolicy` | Deterministic scope resolution (user-customizable, preserved across seed restarts) |
|
|
222
222
|
| **Identity verification** | `identityUrl`, `identityMethod`, `identityHeaders`, `identityBody`, `identityResponsePaths`, `identityFormat`, `identityOkField` | Data-driven identity verifier fetches human-readable account info after token exchange |
|
|
223
223
|
| **Injection templates** | `injectionTemplates` | Auto-applied credential injection rules for the script proxy |
|
|
@@ -265,10 +265,11 @@ This replaces provider-specific handlers — any provider in the registry can be
|
|
|
265
265
|
### Adding a New OAuth Provider
|
|
266
266
|
|
|
267
267
|
1. **Add seed data** to `PROVIDER_SEED_DATA` in `assistant/src/oauth/seed-providers.ts`:
|
|
268
|
-
- Set protocol fields: `authUrl`, `tokenUrl`, `defaultScopes`, `scopePolicy`, `
|
|
268
|
+
- Set protocol fields: `authUrl`, `tokenUrl`, `defaultScopes`, `scopePolicy`, `loopbackPort`.
|
|
269
269
|
- Set identity verification: `identityUrl`, `identityMethod`, `identityHeaders`, `identityResponsePaths`, `identityFormat`.
|
|
270
270
|
- Set injection templates: `injectionTemplates` for providers whose tokens should be auto-injected by the script proxy.
|
|
271
271
|
- Set setup metadata: `displayName`, `dashboardUrl`, `appType` enable the generic OAuth setup skill to guide users through app creation.
|
|
272
|
+
- Note: callback transport (`loopback` or `gateway`) is chosen per-flow by the caller, not per-provider. All providers support both transports.
|
|
272
273
|
2. **Alternatively, register dynamically** via the CLI: `assistant oauth providers register <key> --auth-url ... --token-url ...`.
|
|
273
274
|
3. **No handler code needed** — the generic `oauth_connect_start` handler and the connect orchestrator handle the flow automatically.
|
|
274
275
|
|
package/knip.json
CHANGED
|
@@ -3,7 +3,9 @@
|
|
|
3
3
|
"src/**/*.test.ts",
|
|
4
4
|
"src/**/__tests__/**/*.ts",
|
|
5
5
|
"scripts/**/*.ts",
|
|
6
|
-
"src/daemon/main.ts"
|
|
6
|
+
"src/daemon/main.ts",
|
|
7
|
+
"src/messaging/providers/*/types.ts",
|
|
8
|
+
"src/messaging/providers/*/client.ts"
|
|
7
9
|
],
|
|
8
10
|
"project": ["src/**/*.ts", "src/**/*.tsx", "scripts/**/*.ts"],
|
|
9
11
|
"ignoreDependencies": [
|
package/openapi.yaml
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
openapi: 3.0.0
|
|
4
4
|
info:
|
|
5
5
|
title: Vellum Assistant API
|
|
6
|
-
version: 0.5.
|
|
6
|
+
version: 0.5.15
|
|
7
7
|
description: Auto-generated OpenAPI specification for the Vellum Assistant runtime HTTP server.
|
|
8
8
|
servers:
|
|
9
9
|
- url: http://127.0.0.1:7821
|
|
@@ -1855,47 +1855,6 @@ paths:
|
|
|
1855
1855
|
required:
|
|
1856
1856
|
- provider
|
|
1857
1857
|
additionalProperties: false
|
|
1858
|
-
/v1/config/permissions/skip:
|
|
1859
|
-
get:
|
|
1860
|
-
operationId: config_permissions_skip_get
|
|
1861
|
-
summary: Get permission-skip flag
|
|
1862
|
-
description: Return whether dangerouslySkipPermissions is enabled.
|
|
1863
|
-
tags:
|
|
1864
|
-
- config
|
|
1865
|
-
responses:
|
|
1866
|
-
"200":
|
|
1867
|
-
description: Successful response
|
|
1868
|
-
content:
|
|
1869
|
-
application/json:
|
|
1870
|
-
schema:
|
|
1871
|
-
type: object
|
|
1872
|
-
properties:
|
|
1873
|
-
enabled:
|
|
1874
|
-
type: boolean
|
|
1875
|
-
required:
|
|
1876
|
-
- enabled
|
|
1877
|
-
additionalProperties: false
|
|
1878
|
-
put:
|
|
1879
|
-
operationId: config_permissions_skip_put
|
|
1880
|
-
summary: Set permission-skip flag
|
|
1881
|
-
description: Enable or disable dangerouslySkipPermissions.
|
|
1882
|
-
tags:
|
|
1883
|
-
- config
|
|
1884
|
-
responses:
|
|
1885
|
-
"200":
|
|
1886
|
-
description: Successful response
|
|
1887
|
-
requestBody:
|
|
1888
|
-
required: true
|
|
1889
|
-
content:
|
|
1890
|
-
application/json:
|
|
1891
|
-
schema:
|
|
1892
|
-
type: object
|
|
1893
|
-
properties:
|
|
1894
|
-
enabled:
|
|
1895
|
-
type: boolean
|
|
1896
|
-
required:
|
|
1897
|
-
- enabled
|
|
1898
|
-
additionalProperties: false
|
|
1899
1858
|
/v1/config/platform:
|
|
1900
1859
|
get:
|
|
1901
1860
|
operationId: config_platform_get
|
|
@@ -6759,7 +6718,7 @@ paths:
|
|
|
6759
6718
|
buckets:
|
|
6760
6719
|
type: array
|
|
6761
6720
|
items: {}
|
|
6762
|
-
description:
|
|
6721
|
+
description: Usage bucket objects
|
|
6763
6722
|
required:
|
|
6764
6723
|
- buckets
|
|
6765
6724
|
additionalProperties: false
|
|
@@ -6776,6 +6735,15 @@ paths:
|
|
|
6776
6735
|
schema:
|
|
6777
6736
|
type: integer
|
|
6778
6737
|
description: End epoch millis (required)
|
|
6738
|
+
- name: granularity
|
|
6739
|
+
in: query
|
|
6740
|
+
required: false
|
|
6741
|
+
schema:
|
|
6742
|
+
type: string
|
|
6743
|
+
enum:
|
|
6744
|
+
- daily
|
|
6745
|
+
- hourly
|
|
6746
|
+
description: 'Bucket granularity: "daily" (default) or "hourly"'
|
|
6779
6747
|
/v1/usage/totals:
|
|
6780
6748
|
get:
|
|
6781
6749
|
operationId: usage_totals_get
|
package/package.json
CHANGED
|
@@ -1,85 +1,20 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Integration tests for assistant feature flag
|
|
3
|
-
* skill_load, and session-skill-tools projection layers.
|
|
2
|
+
* Integration tests for assistant feature flag resolver.
|
|
4
3
|
*
|
|
5
4
|
* Covers:
|
|
6
|
-
* - Flag OFF blocks all exposure paths
|
|
7
5
|
* - Missing persisted value falls back to code default
|
|
8
6
|
* - Protected feature-flags.json is the sole override mechanism
|
|
9
7
|
* - Undeclared keys default to enabled
|
|
10
8
|
*/
|
|
11
|
-
import {
|
|
12
|
-
import { join } from "node:path";
|
|
13
|
-
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
|
9
|
+
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
|
|
14
10
|
|
|
15
11
|
// ---------------------------------------------------------------------------
|
|
16
|
-
// Test-scoped
|
|
12
|
+
// Test-scoped config state
|
|
17
13
|
// ---------------------------------------------------------------------------
|
|
18
14
|
|
|
19
|
-
const TEST_DIR = process.env.VELLUM_WORKSPACE_DIR!;
|
|
20
|
-
|
|
21
|
-
let currentConfig: Record<string, unknown> = {
|
|
22
|
-
services: {
|
|
23
|
-
inference: {
|
|
24
|
-
mode: "your-own",
|
|
25
|
-
provider: "anthropic",
|
|
26
|
-
model: "claude-opus-4-6",
|
|
27
|
-
},
|
|
28
|
-
"image-generation": {
|
|
29
|
-
mode: "your-own",
|
|
30
|
-
provider: "gemini",
|
|
31
|
-
model: "gemini-3.1-flash-image-preview",
|
|
32
|
-
},
|
|
33
|
-
"web-search": { mode: "your-own", provider: "inference-provider-native" },
|
|
34
|
-
},
|
|
35
|
-
};
|
|
36
|
-
|
|
37
15
|
const DECLARED_FLAG_ID = "contacts";
|
|
38
16
|
const DECLARED_FLAG_KEY = DECLARED_FLAG_ID;
|
|
39
|
-
const DECLARED_SKILL_ID = "contacts";
|
|
40
|
-
|
|
41
|
-
const noopLogger = new Proxy({} as Record<string, unknown>, {
|
|
42
|
-
get: (_target, prop) => (prop === "child" ? () => noopLogger : () => {}),
|
|
43
|
-
});
|
|
44
|
-
|
|
45
|
-
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
46
|
-
const realLogger = require("../util/logger.js");
|
|
47
|
-
mock.module("../util/logger.js", () => ({
|
|
48
|
-
...realLogger,
|
|
49
|
-
getLogger: () => noopLogger,
|
|
50
|
-
getCliLogger: () => noopLogger,
|
|
51
|
-
truncateForLog: (v: string) => v,
|
|
52
|
-
initLogger: () => {},
|
|
53
|
-
pruneOldLogFiles: () => 0,
|
|
54
|
-
}));
|
|
55
17
|
|
|
56
|
-
mock.module("../config/loader.js", () => ({
|
|
57
|
-
getConfig: () => currentConfig,
|
|
58
|
-
loadConfig: () => currentConfig,
|
|
59
|
-
loadRawConfig: () => ({}),
|
|
60
|
-
saveConfig: () => {},
|
|
61
|
-
saveRawConfig: () => {},
|
|
62
|
-
invalidateConfigCache: () => {},
|
|
63
|
-
getNestedValue: () => undefined,
|
|
64
|
-
setNestedValue: () => {},
|
|
65
|
-
}));
|
|
66
|
-
|
|
67
|
-
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
68
|
-
const realUserReference = require("../prompts/user-reference.js");
|
|
69
|
-
mock.module("../prompts/user-reference.js", () => ({
|
|
70
|
-
...realUserReference,
|
|
71
|
-
resolveUserReference: () => "TestUser",
|
|
72
|
-
resolveUserPronouns: () => null,
|
|
73
|
-
}));
|
|
74
|
-
|
|
75
|
-
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
76
|
-
const realCredentialMetadataStore = require("../tools/credentials/metadata-store.js");
|
|
77
|
-
mock.module("../tools/credentials/metadata-store.js", () => ({
|
|
78
|
-
...realCredentialMetadataStore,
|
|
79
|
-
listCredentialMetadata: () => [],
|
|
80
|
-
}));
|
|
81
|
-
|
|
82
|
-
const { buildSystemPrompt } = await import("../prompts/system-prompt.js");
|
|
83
18
|
const { isAssistantFeatureFlagEnabled, _setOverridesForTesting } =
|
|
84
19
|
await import("../config/assistant-feature-flags.js");
|
|
85
20
|
const { skillFlagKey } = await import("../config/skill-state.js");
|
|
@@ -90,319 +25,12 @@ const { skillFlagKey } = await import("../config/skill-state.js");
|
|
|
90
25
|
|
|
91
26
|
beforeEach(() => {
|
|
92
27
|
_setOverridesForTesting({});
|
|
93
|
-
currentConfig = {
|
|
94
|
-
services: {
|
|
95
|
-
inference: {
|
|
96
|
-
mode: "your-own",
|
|
97
|
-
provider: "anthropic",
|
|
98
|
-
model: "claude-opus-4-6",
|
|
99
|
-
},
|
|
100
|
-
"image-generation": {
|
|
101
|
-
mode: "your-own",
|
|
102
|
-
provider: "gemini",
|
|
103
|
-
model: "gemini-3.1-flash-image-preview",
|
|
104
|
-
},
|
|
105
|
-
"web-search": { mode: "your-own", provider: "inference-provider-native" },
|
|
106
|
-
},
|
|
107
|
-
};
|
|
108
28
|
});
|
|
109
29
|
|
|
110
30
|
afterEach(() => {
|
|
111
31
|
_setOverridesForTesting({});
|
|
112
32
|
});
|
|
113
33
|
|
|
114
|
-
// ---------------------------------------------------------------------------
|
|
115
|
-
// Helpers
|
|
116
|
-
// ---------------------------------------------------------------------------
|
|
117
|
-
|
|
118
|
-
function createSkillOnDisk(
|
|
119
|
-
id: string,
|
|
120
|
-
name: string,
|
|
121
|
-
description: string,
|
|
122
|
-
featureFlag?: string,
|
|
123
|
-
): void {
|
|
124
|
-
const skillsDir = join(TEST_DIR, "skills");
|
|
125
|
-
mkdirSync(join(skillsDir, id), { recursive: true });
|
|
126
|
-
const ffBlock = featureFlag
|
|
127
|
-
? `\nmetadata: {"vellum":{"feature-flag":"${featureFlag}"}}`
|
|
128
|
-
: "";
|
|
129
|
-
writeFileSync(
|
|
130
|
-
join(skillsDir, id, "SKILL.md"),
|
|
131
|
-
`---\nname: "${name}"\ndescription: "${description}"${ffBlock}\n---\n\nInstructions for ${id}.\n`,
|
|
132
|
-
);
|
|
133
|
-
const indexPath = join(skillsDir, "SKILLS.md");
|
|
134
|
-
const existing = existsSync(indexPath)
|
|
135
|
-
? readFileSync(indexPath, "utf-8")
|
|
136
|
-
: "";
|
|
137
|
-
writeFileSync(indexPath, existing + `- ${id}\n`);
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
// ---------------------------------------------------------------------------
|
|
141
|
-
// System prompt — assistant feature flag filtering
|
|
142
|
-
// ---------------------------------------------------------------------------
|
|
143
|
-
|
|
144
|
-
describe("buildSystemPrompt assistant feature flag filtering", () => {
|
|
145
|
-
test("flag OFF skill does not appear in skills catalog", () => {
|
|
146
|
-
createSkillOnDisk(
|
|
147
|
-
DECLARED_SKILL_ID,
|
|
148
|
-
"Contacts",
|
|
149
|
-
"Toggle contacts behavior",
|
|
150
|
-
DECLARED_FLAG_ID,
|
|
151
|
-
);
|
|
152
|
-
createSkillOnDisk(
|
|
153
|
-
"browser",
|
|
154
|
-
"Browser",
|
|
155
|
-
"Web browsing automation",
|
|
156
|
-
"browser",
|
|
157
|
-
);
|
|
158
|
-
|
|
159
|
-
_setOverridesForTesting({
|
|
160
|
-
[DECLARED_FLAG_KEY]: false,
|
|
161
|
-
browser: true,
|
|
162
|
-
});
|
|
163
|
-
|
|
164
|
-
currentConfig = {
|
|
165
|
-
services: {
|
|
166
|
-
inference: {
|
|
167
|
-
mode: "your-own",
|
|
168
|
-
provider: "anthropic",
|
|
169
|
-
model: "claude-opus-4-6",
|
|
170
|
-
},
|
|
171
|
-
"image-generation": {
|
|
172
|
-
mode: "your-own",
|
|
173
|
-
provider: "gemini",
|
|
174
|
-
model: "gemini-3.1-flash-image-preview",
|
|
175
|
-
},
|
|
176
|
-
"web-search": {
|
|
177
|
-
mode: "your-own",
|
|
178
|
-
provider: "inference-provider-native",
|
|
179
|
-
},
|
|
180
|
-
},
|
|
181
|
-
};
|
|
182
|
-
|
|
183
|
-
const result = buildSystemPrompt();
|
|
184
|
-
|
|
185
|
-
// browser is explicitly enabled, declared flagged skill is explicitly off
|
|
186
|
-
expect(result).toContain("**browser**");
|
|
187
|
-
expect(result).not.toContain(`**${DECLARED_SKILL_ID}**`);
|
|
188
|
-
});
|
|
189
|
-
|
|
190
|
-
test("contacts visible but email-channel hidden when no flag overrides set (contacts defaults true, email-channel defaults false)", () => {
|
|
191
|
-
createSkillOnDisk(
|
|
192
|
-
DECLARED_SKILL_ID,
|
|
193
|
-
"Contacts",
|
|
194
|
-
"Toggle contacts behavior",
|
|
195
|
-
DECLARED_FLAG_ID,
|
|
196
|
-
);
|
|
197
|
-
createSkillOnDisk(
|
|
198
|
-
"email-channel",
|
|
199
|
-
"Email Channel",
|
|
200
|
-
"Email channel setup",
|
|
201
|
-
"email-channel",
|
|
202
|
-
);
|
|
203
|
-
|
|
204
|
-
currentConfig = {
|
|
205
|
-
services: {
|
|
206
|
-
inference: {
|
|
207
|
-
mode: "your-own",
|
|
208
|
-
provider: "anthropic",
|
|
209
|
-
model: "claude-opus-4-6",
|
|
210
|
-
},
|
|
211
|
-
"image-generation": {
|
|
212
|
-
mode: "your-own",
|
|
213
|
-
provider: "gemini",
|
|
214
|
-
model: "gemini-3.1-flash-image-preview",
|
|
215
|
-
},
|
|
216
|
-
"web-search": {
|
|
217
|
-
mode: "your-own",
|
|
218
|
-
provider: "inference-provider-native",
|
|
219
|
-
},
|
|
220
|
-
},
|
|
221
|
-
};
|
|
222
|
-
|
|
223
|
-
const result = buildSystemPrompt();
|
|
224
|
-
|
|
225
|
-
// contacts defaults to true, email-channel defaults to false
|
|
226
|
-
expect(result).toContain(`**${DECLARED_SKILL_ID}**`);
|
|
227
|
-
expect(result).not.toContain("**email-channel**");
|
|
228
|
-
});
|
|
229
|
-
|
|
230
|
-
test("flagged-off skills hidden when all flags are OFF", () => {
|
|
231
|
-
createSkillOnDisk(
|
|
232
|
-
DECLARED_SKILL_ID,
|
|
233
|
-
"Contacts",
|
|
234
|
-
"Toggle contacts behavior",
|
|
235
|
-
DECLARED_FLAG_ID,
|
|
236
|
-
);
|
|
237
|
-
createSkillOnDisk(
|
|
238
|
-
"email-channel",
|
|
239
|
-
"Email Channel",
|
|
240
|
-
"Email channel setup",
|
|
241
|
-
"email-channel",
|
|
242
|
-
);
|
|
243
|
-
|
|
244
|
-
_setOverridesForTesting({
|
|
245
|
-
[DECLARED_FLAG_KEY]: false,
|
|
246
|
-
"email-channel": false,
|
|
247
|
-
});
|
|
248
|
-
|
|
249
|
-
currentConfig = {
|
|
250
|
-
services: {
|
|
251
|
-
inference: {
|
|
252
|
-
mode: "your-own",
|
|
253
|
-
provider: "anthropic",
|
|
254
|
-
model: "claude-opus-4-6",
|
|
255
|
-
},
|
|
256
|
-
"image-generation": {
|
|
257
|
-
mode: "your-own",
|
|
258
|
-
provider: "gemini",
|
|
259
|
-
model: "gemini-3.1-flash-image-preview",
|
|
260
|
-
},
|
|
261
|
-
"web-search": {
|
|
262
|
-
mode: "your-own",
|
|
263
|
-
provider: "inference-provider-native",
|
|
264
|
-
},
|
|
265
|
-
},
|
|
266
|
-
};
|
|
267
|
-
|
|
268
|
-
const result = buildSystemPrompt();
|
|
269
|
-
|
|
270
|
-
expect(result).not.toContain(`**${DECLARED_SKILL_ID}**`);
|
|
271
|
-
expect(result).not.toContain("**email-channel**");
|
|
272
|
-
});
|
|
273
|
-
|
|
274
|
-
test("file-based overrides control visibility", () => {
|
|
275
|
-
createSkillOnDisk(
|
|
276
|
-
DECLARED_SKILL_ID,
|
|
277
|
-
"Contacts",
|
|
278
|
-
"Toggle contacts behavior",
|
|
279
|
-
DECLARED_FLAG_ID,
|
|
280
|
-
);
|
|
281
|
-
|
|
282
|
-
_setOverridesForTesting({ [DECLARED_FLAG_KEY]: true });
|
|
283
|
-
|
|
284
|
-
currentConfig = {
|
|
285
|
-
services: {
|
|
286
|
-
inference: {
|
|
287
|
-
mode: "your-own",
|
|
288
|
-
provider: "anthropic",
|
|
289
|
-
model: "claude-opus-4-6",
|
|
290
|
-
},
|
|
291
|
-
"image-generation": {
|
|
292
|
-
mode: "your-own",
|
|
293
|
-
provider: "gemini",
|
|
294
|
-
model: "gemini-3.1-flash-image-preview",
|
|
295
|
-
},
|
|
296
|
-
"web-search": {
|
|
297
|
-
mode: "your-own",
|
|
298
|
-
provider: "inference-provider-native",
|
|
299
|
-
},
|
|
300
|
-
},
|
|
301
|
-
};
|
|
302
|
-
|
|
303
|
-
const result = buildSystemPrompt();
|
|
304
|
-
|
|
305
|
-
expect(result).toContain(`**${DECLARED_SKILL_ID}**`);
|
|
306
|
-
});
|
|
307
|
-
|
|
308
|
-
test("persisted overrides for undeclared flags are respected", () => {
|
|
309
|
-
createSkillOnDisk(
|
|
310
|
-
"browser",
|
|
311
|
-
"Browser",
|
|
312
|
-
"Web browsing automation",
|
|
313
|
-
"browser",
|
|
314
|
-
);
|
|
315
|
-
|
|
316
|
-
_setOverridesForTesting({ browser: false });
|
|
317
|
-
|
|
318
|
-
currentConfig = {
|
|
319
|
-
services: {
|
|
320
|
-
inference: {
|
|
321
|
-
mode: "your-own",
|
|
322
|
-
provider: "anthropic",
|
|
323
|
-
model: "claude-opus-4-6",
|
|
324
|
-
},
|
|
325
|
-
"image-generation": {
|
|
326
|
-
mode: "your-own",
|
|
327
|
-
provider: "gemini",
|
|
328
|
-
model: "gemini-3.1-flash-image-preview",
|
|
329
|
-
},
|
|
330
|
-
"web-search": {
|
|
331
|
-
mode: "your-own",
|
|
332
|
-
provider: "inference-provider-native",
|
|
333
|
-
},
|
|
334
|
-
},
|
|
335
|
-
};
|
|
336
|
-
|
|
337
|
-
const result = buildSystemPrompt();
|
|
338
|
-
|
|
339
|
-
// browser declares featureFlag: "browser" and the user
|
|
340
|
-
// explicitly disabled it — that override must be honored.
|
|
341
|
-
expect(result).not.toContain("**browser**");
|
|
342
|
-
});
|
|
343
|
-
|
|
344
|
-
test("declared flags with no persisted override use registry default", () => {
|
|
345
|
-
createSkillOnDisk(
|
|
346
|
-
"browser",
|
|
347
|
-
"Browser",
|
|
348
|
-
"Web browsing automation",
|
|
349
|
-
"browser",
|
|
350
|
-
);
|
|
351
|
-
|
|
352
|
-
currentConfig = {
|
|
353
|
-
services: {
|
|
354
|
-
inference: {
|
|
355
|
-
mode: "your-own",
|
|
356
|
-
provider: "anthropic",
|
|
357
|
-
model: "claude-opus-4-6",
|
|
358
|
-
},
|
|
359
|
-
"image-generation": {
|
|
360
|
-
mode: "your-own",
|
|
361
|
-
provider: "gemini",
|
|
362
|
-
model: "gemini-3.1-flash-image-preview",
|
|
363
|
-
},
|
|
364
|
-
"web-search": {
|
|
365
|
-
mode: "your-own",
|
|
366
|
-
provider: "inference-provider-native",
|
|
367
|
-
},
|
|
368
|
-
},
|
|
369
|
-
};
|
|
370
|
-
|
|
371
|
-
const result = buildSystemPrompt();
|
|
372
|
-
|
|
373
|
-
// browser is declared in the registry with defaultEnabled: true
|
|
374
|
-
expect(result).toContain("**browser**");
|
|
375
|
-
});
|
|
376
|
-
|
|
377
|
-
test("skill without featureFlag is never flag-gated", () => {
|
|
378
|
-
createSkillOnDisk("my-skill", "My Skill", "A skill without feature flag");
|
|
379
|
-
|
|
380
|
-
currentConfig = {
|
|
381
|
-
services: {
|
|
382
|
-
inference: {
|
|
383
|
-
mode: "your-own",
|
|
384
|
-
provider: "anthropic",
|
|
385
|
-
model: "claude-opus-4-6",
|
|
386
|
-
},
|
|
387
|
-
"image-generation": {
|
|
388
|
-
mode: "your-own",
|
|
389
|
-
provider: "gemini",
|
|
390
|
-
model: "gemini-3.1-flash-image-preview",
|
|
391
|
-
},
|
|
392
|
-
"web-search": {
|
|
393
|
-
mode: "your-own",
|
|
394
|
-
provider: "inference-provider-native",
|
|
395
|
-
},
|
|
396
|
-
},
|
|
397
|
-
};
|
|
398
|
-
|
|
399
|
-
const result = buildSystemPrompt();
|
|
400
|
-
|
|
401
|
-
// Skills without featureFlag declared are never gated — always pass through
|
|
402
|
-
expect(result).toContain("**my-skill**");
|
|
403
|
-
});
|
|
404
|
-
});
|
|
405
|
-
|
|
406
34
|
// ---------------------------------------------------------------------------
|
|
407
35
|
// Resolver unit tests (within integration context)
|
|
408
36
|
// ---------------------------------------------------------------------------
|
|
@@ -185,7 +185,10 @@ describe("CesRpcCredentialBackend", () => {
|
|
|
185
185
|
const result = await backend.list();
|
|
186
186
|
|
|
187
187
|
expect(callFn).toHaveBeenCalledWith(CesRpcMethod.ListCredentials, {});
|
|
188
|
-
expect(result).toEqual({
|
|
188
|
+
expect(result).toEqual({
|
|
189
|
+
accounts: ["account-a", "account-b"],
|
|
190
|
+
unreachable: false,
|
|
191
|
+
});
|
|
189
192
|
});
|
|
190
193
|
|
|
191
194
|
test("returns unreachable when RPC call throws", async () => {
|