@holokai/holo-test 1.4.5 → 1.5.0

package/README.md

@@ -1,7 +1,11 @@
 # @holokai/holo-test
 
-Conformance testing framework for Holo provider plugins. Provides fixture-driven testers, vitest contracts, a fluent
-DSL, and a CLI.
+Conformance testing framework for Holo provider and datastore plugins.
+
+The framework owns the test logic. Plugins ship only data: per-protocol scenario files plus an optional minimal SDK
+adapter. A single `runConformance()` call discovers every installed `@holokai/holo-provider-*` and
+`@holokai/holo-datastore-*` package, loads its scenarios, and registers translator + wire + audit + pipeline + sdk
+vitest blocks.
 
 ## Installation
 
@@ -13,161 +17,292 @@ Peer dependencies: `vitest ^3.0.0`, `tsyringe ^4.10.0`
 
 ## Quick Start
 
+A single test file at the repo root (or in any package that depends on the plugins) is enough:
+
 ```typescript
-import {describe, it} from 'vitest';
-import {loadFixtures, runWireContract} from '@holokai/holo-test';
-
-const fixtures = loadFixtures('./tests/fixtures', {plugin: 'openai'});
-
-describe('openai wire conformance', () => {
-    for (const fixture of fixtures) {
-        it(fixture.name, async () => {
-            await runWireContract('openai', fixture);
-        });
-    }
-});
+// tests/conformance.test.ts
+import 'reflect-metadata';
+import {runConformance} from '@holokai/holo-test';
+
+await runConformance();
 ```
 
+`runConformance()` calls `vitest.describe` internally; running this file with `vitest run` produces one suite per
+discovered plugin, with one `describe` per scenario.
+
 ## Testing Modes
 
-| Mode          | What it Tests                                                                      | Function                                     |
-|---------------|------------------------------------------------------------------------------------|----------------------------------------------|
-| **Wire**      | Provider events serialize to correct HTTP wire chunks (SSE/JSON)                   | `testWire()` / `runWireContract()`           |
-| **Audit**     | Worker requests produce correct `ProviderRequest`/`ProviderResponse` audit records | `testAudit()` / `runAuditContract()`         |
-| **Pipeline**  | End-to-end: payload in, events out, text accumulated, done event emitted           | `testPipeline()` / `runPipelineContract()`   |
-| **RoundTrip** | SDK request → HTTP fixture server → SDK response matches expected output           | `testRoundTrip()` / `runRoundTripContract()` |
+| Mode           | What it tests                                                                                                      | Triggered when                                                                         |
+|----------------|--------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------|
+| **Translator** | Plugin translator round-trips between native and Holo canonical types                                              | `scenario.request` and/or `scenario.response` set                                      |
+| **Wire**       | Provider events serialize to the expected HTTP wire (status, headers, body)                                        | `scenario.wire` set + `scenario.providerChunks` populated                              |
+| **Audit**      | Provider events produce the expected `ProviderResponse` audit shape                                                | `scenario.audit` set + `scenario.providerChunks` populated                             |
+| **Pipeline**   | End-to-end provider events → wire chunks → text accumulation; chunk-level shape and `done` terminator are asserted | `scenario.pipeline === true` + `scenario.providerChunks` + `scenario.expectedText` set |
+| **SDK**        | Real provider SDK client → fixture HTTP server → expected SDK result                                               | `scenario.sdk` set + `tests/sdk-adapter.ts` exists                                     |
+| **Datastore**  | Datastore plugin connects, writes a request/response/cost row, can be queried                                      | `tests/datastore-config.ts` exists in the datastore plugin                             |
+| **Live**       | Real provider API call; asserts `protocol.streamEventSequence` via `assertEventSequence`                           | `holo-test-live` vitest project; gated on `isProviderAvailable(family)`                |
+
+Modes activate independently — a scenario can exercise just `wire`+`audit` without setting `sdk` or `pipeline`.
+
+### Pipeline shape assertions
+
+When `scenario.pipeline === true`, the framework collects every chunk delivered to `publishChunk` during the run.
 
-## Fixtures
+- If `scenario.expectedPipelineChunks` is provided, the collected sequence must match exactly.
+- Otherwise, the soft path checks that the chunk count is `> 0` and the final chunk carries `done: true` — enough to
+  catch wire-adapter regressions that drop the terminator.
 
-Create `.fixture.ts` files that export a `FixtureScenario`:
+## Scenarios
+
+Place scenarios under `tests/data/{protocol}/{name}.scenario.ts` in a plugin. Each file default-exports a
+`HoloScenario`:
 
 ```typescript
-import type {FixtureScenario} from '@holokai/holo-test';
+import type {HoloScenario} from '@holokai/holo-test';
 
-const fixture: FixtureScenario = {
+const scenario: HoloScenario = {
     name: 'simple chat completion',
-    plugin: 'openai',
     protocol: 'openai.chatCompletions',
     streaming: false,
 
-    providerChunks: [/* mock provider response objects */],
+    // Translator round-trip (optional)
+    request: {
+        native: {model: 'gpt-4o', messages: [{role: 'user', content: 'Hi'}]},
+        holo: {model: 'gpt-4o', messages: [{role: 'user', content: 'Hi'}]},
+    },
+    response: {
+        native: {/* native ChatCompletion */},
+        holo: {/* partial HoloResponse */},
+    },
 
+    // Provider event sequence consumed by wire/audit/pipeline/sdk
+    providerChunks: [/* native chunk(s) */],
     expectedText: 'Hello, world!',
-    expectedWire: ['{"id":"chatcmpl-...}'],
-    expectedStatus: 200,
-    expectedHeaders: {'content-type': 'application/json'},
 
-    expectedAudit: {
+    // Wire-conformance
+    wire: {
+        expectedStatus: 200,
+        expectedHeaders: {'content-type': 'application/json'},
+        expectedWire: ['{"id":"chatcmpl-..."}'],
+    },
+
+    // Audit-conformance
+    audit: {
         access_model: 'gpt-4o',
         input_tokens: 10,
         output_tokens: 5,
         status: 'success',
     },
 
-    tags: ['chat', 'non-streaming'],
+    // Pipeline opt-in
+    pipeline: true,
+
+    // SDK round-trip (requires tests/sdk-adapter.ts)
+    sdk: {
+        request: {model: 'gpt-4o', messages: [{role: 'user', content: 'Hi'}]},
+        expectedResult: {choices: [{message: {content: 'Hello, world!'}}]},
+    },
 };
 
-export default fixture;
+export default scenario;
 ```
 
-Load fixtures:
+### Loader Invariants
 
-```typescript
-import {loadFixtures} from '@holokai/holo-test';
+- Directory name **must** equal `scenario.protocol` — the loader throws otherwise. So a scenario with
+  `protocol: 'claude.messages'` must live under `tests/data/claude.messages/`.
+- Files matching `*.scenario.ts` (or `.scenario.js`) are picked up automatically; no central registration needed.
+- The default export (or named `scenario` export) is what the loader reads.
+- Every scenario's `protocol` must appear in the plugin's declared `protocols` map; mismatches throw at discovery time.
 
-// All fixtures
-const all = loadFixtures('./tests/fixtures');
+### Canonical streaming fixtures
 
-// Filtered
-const streaming = loadFixtures('./tests/fixtures', {plugin: 'openai', tag: 'streaming'});
-```
+`packages/holo-test/holo/streaming/*.holo.ts` declares the canonical Holo-pivot streaming sequences used by
+`registerStreamingPerPlugin` and `registerStreamingCross`. Each fixture opens with a `message_start` chunk and closes
+with a `message_stop` chunk, mirroring real provider SSE lifecycle. Per-plugin streaming round-trip is strict:
+translators must emit and consume those lifecycle chunks to round-trip cleanly.
+
+A plugin that doesn't yet emit/consume native equivalents of `message_start` / `message_stop` (e.g. Claude
+`message_start` envelopes, OpenAI `[DONE]` terminators, Bedrock `messageStart`/`messageStop`) should pin its scenario
+with `expectedFailure: { toHolo: true, fromHolo: true, reason: '...' }` until the gap is closed.
+
+### Strip helpers
+
+`registerStreamingPerPlugin` and `registerStreamingCross` strip volatile fields before comparing chunk arrays:
+
+| Helper                  | Strips                                                             | Where applied                          |
+|-------------------------|--------------------------------------------------------------------|----------------------------------------|
+| `stripProviderMetadata` | `delta.provider`, `delta.provider_delta` (translator pass-through) | Per-plugin round-trip + cross-provider |
+| `stripCrossVolatile`    | Top-level `id` / `created` / `model`, message-level `id` / `model` | Cross-provider only                    |
 
-## Suite DSL
+Per-plugin (same-provider in/out) round-trip stays strict on `id` / `created` / `model` — a translator that drops or
+fabricates them now fails the round-trip immediately rather than being masked.
 
-Fluent builder for running multiple test modes:
+### Streaming event sequence assertion
+
+`assertEventSequence(events, sequence)` walks the `ProviderEvent` stream and verifies it satisfies the protocol's
+declared `streamEventSequence` (`{ ordered: string[], repeatable: string[] }`). It returns an `AssertionError` for
+missing required events, out-of-order non-repeatable events, or skipped required events; it is exercised via
+`runLiveTest` (real provider call), not in the offline conformance run. The native event-type extractor handles both
+top-level `.type` shapes (Claude, OpenAI Responses, Gemini structured) and discriminator-key shapes (Bedrock-style
+`{contentBlockDelta: {...}}`).
+
+### Pinning Known Losses
+
+When a translator/wire/audit asymmetry is known and accepted (e.g., a provider drops a Holo content type that has no
+native equivalent), record it on the scenario rather than silencing the test:
 
 ```typescript
-import {suite} from '@holokai/holo-test';
-
-await suite()
-    .forPlugin('openai')
-    .fixtureDir('./tests/fixtures')
-    .wire()
-    .audit()
-    .pipeline()
-    .verbose()
-    .run();
+scenario.expectedFailure = {
+    response: {fromHolo: true, roundTrip: true}, // Holo→native loses something
+    reason: 'Claude response blocks have no equivalent for Holo image content',
+};
 ```
 
-## Vitest Contracts
+Failing-as-expected modes flip to `it.fails`, so CI still fails if the scenario unexpectedly *starts* passing.
+
+## SDK Adapter
 
-Import contract functions directly into vitest test files:
+If you want to exercise SDK round-trip (real provider SDK client → fixture HTTP server → assert), drop a
+`tests/sdk-adapter.ts` in the plugin:
 
 ```typescript
-import {runWireContract, runAuditContract, runPipelineContract} from '@holokai/holo-test';
+import OpenAI from 'openai';
+import type {HoloScenario, ScenarioSdkAdapter} from '@holokai/holo-test';
+
+const adapter: ScenarioSdkAdapter = {
+    family: 'openai',
 
-describe('openai conformance', () => {
-    it('wire: simple chat', async () => {
-        await runWireContract('openai', fixture);
-    });
+    async call(scenario: HoloScenario, port: number) {
+        const client = new OpenAI({apiKey: 'test', baseURL: `http://localhost:${port}/v1`});
+        return client.chat.completions.create(scenario.sdk!.request as never);
+    },
 
-    it('audit: simple chat', async () => {
-        await runAuditContract('openai', fixture);
-    });
+    routes(scenario: HoloScenario) {
+        if (scenario.protocol === 'openai.chatCompletions') {
+            return {method: 'POST', path: '/v1/chat/completions'};
+        }
+        return undefined;
+    },
+};
 
-    it('pipeline: simple chat', async () => {
-        await runPipelineContract('openai', fixture);
-    });
-});
+export default adapter;
 ```
 
-## CLI
+The runner only loads `sdk-adapter.ts` when at least one scenario sets `sdk`, so plugins that don't yet care about
+SDK round-trip can omit the file. See `plugins/holo-provider-openai/tests/sdk-adapter.ts` for the canonical example.
 
-```bash
-npx holo-test --fixture-dir ./tests/fixtures --plugin openai
-npx holo-test --fixture-dir ./tests/fixtures --plugin claude --tag streaming --verbose
+## Datastore Configs
+
+Datastore plugins (`@holokai/holo-datastore-*`) supply a `tests/datastore-config.ts` instead of scenarios:
+
+```typescript
+import type {DatastoreConfig} from '@holokai/holo-test';
+
+const config: DatastoreConfig = {
+    connectionConfig: {/* host, port, auth, schema, ... */},
+    requestTable: 'provider_requests',
+    responseTable: 'provider_responses',
+    costTable: 'provider_response_costs',
+    cleanup: async (insertedIds) => {/* delete inserted rows */},
+};
+
+export default config;
+```
+
+`runConformance()` exercises connect, write, and read on each datastore plugin that has a config file.
+
+## Authoring a New Plugin Test
+
+1. Add `tests/setup.ts` with a single line: `import 'reflect-metadata';`
+2. Drop one or more `tests/data/{protocol}/*.scenario.ts` files (default-export a `HoloScenario`).
+3. (Optional) add `tests/sdk-adapter.ts` exporting a default `ScenarioSdkAdapter` if you want SDK round-trip
+   coverage.
+4. Make sure the plugin's package depends on `@holokai/holo-test` (devDependency) and that the repo-level
+   `runConformance()` test file picks up the new package via `discoverPlugins()`.
+
+`plugins/holo-provider-claude/tests/data/claude.messages/simple.scenario.ts` is a good template for translator + wire +
+audit coverage; `plugins/holo-provider-openai/tests/sdk-adapter.ts` is the canonical adapter example.
+
+## Composing Manually
+
+For unusual cases — e.g. a single scenario you want to register outside the auto-discovery flow — the per-mode
+register functions are exported:
+
+```typescript
+import {
+    loadPlugin,
+    loadScenarios,
+    registerTranslatorTests,
+    registerWireTests,
+    registerAuditTests,
+    registerPipelineTests,
+    registerSdkTests,
+} from '@holokai/holo-test';
+
+const plugin = await loadPlugin('openai');
+const scenarios = await loadScenarios('openai');
+
+for (const scenario of scenarios.filter(s => s.name === 'tool calling')) {
+    registerTranslatorTests(plugin.translator, scenario);
+    registerWireTests(plugin, scenario);
+    registerAuditTests(plugin, scenario);
+    registerPipelineTests('openai', plugin, scenario);
+    registerSdkTests(plugin, scenario, undefined);
+}
 ```
 
+Prefer `runConformance()` for normal use; the manual path is an escape hatch.
+
 ## Live Testing
 
-Test against real provider APIs:
+Tests against real provider APIs (separate from conformance — no fixture server, real network). `runLiveTest` loads the
+plugin, makes a real call, collects the resulting `ProviderEvent` stream, and runs `assertEventSequence` against the
+protocol's declared `streamEventSequence`. Failures surface in the returned `errors[]`.
 
 ```typescript
 import {runLiveTest, isProviderAvailable} from '@holokai/holo-test';
 
 if (isProviderAvailable('openai')) {
-    const results = await runLiveTest({
-        families: ['openai'],
-        timeout: 30000,
-    });
-    console.log(results);
+    const result = await runLiveTest('openai', /* streaming */ true);
+    console.log(result.passed, result.errors);
 }
 ```
 
+This package's own `tests/live/live.test.ts` exercises `runLiveTest` against every plugin family in both modes; it runs
+under the `holo-test-live` vitest project, which `npm run test:live` chains after `app-aio` and `sdk-live`. Tests
+auto-skip when the family's API key (or `OLLAMA_HOST`) isn't set.
+
 Environment variables for live tests:
 
-- `OPENAI_API_KEY`, `CLAUDE_API_KEY`, `GEMINI_API_KEY`, `OLLAMA_HOST`
-- `OPENAI_MODEL`, `CLAUDE_MODEL`, etc. (override default test models)
+- `OPENAI_API_KEY`, `ANTHROPIC_API_KEY` (alias for `CLAUDE_API_KEY`), `GOOGLE_API_KEY` (alias for `GEMINI_API_KEY`),
+  `BEDROCK_API_KEY`, `OLLAMA_HOST`
+- `OPENAI_TEST_MODEL`, `CLAUDE_TEST_MODEL`, etc. (override default test models)
 
 ## Helpers
 
 ```typescript
 import {
     loadPlugin,                  // Load and initialize a plugin by family name
+    loadScenarios,               // Load scenarios for a single plugin
+    loadSdkAdapter,              // Load tests/sdk-adapter.ts for a plugin
+    loadDatastoreConfig,         // Load tests/datastore-config.ts for a datastore plugin
     parseSseBody,                // Parse SSE response body into frames
     parseNdjsonBody,             // Parse NDJSON response body
     createMockFetch,             // Mock fetch for unit tests
     createSseMockFetch,          // Mock SSE streaming responses
     collectStreamText,           // Extract all text from a stream
     collectStreamEvents,         // Collect all events from a stream
-    HttpFixtureServer,           // Mock HTTP server serving fixture responses
-    createTestWorkerRequest,     // Build a HoloWorkerRequest from a fixture
-    createTestEnvelope,          // Build a WorkerResponseEnvelope from a fixture
     discoverProviders,           // Discover available providers via gateway
     assertTestResult,            // Assert a TestResult passed (throws on failure)
+    createTestPlugin,            // Build a temporary plugin tarball for upload/install tests
+    packPluginFixture,           // Pack an existing plugin directory into a tarball
 } from '@holokai/holo-test';
 ```
 
+`createTestPlugin` / `packPluginFixture` are for *runtime* plugin tests (upload/install/lifecycle), not for
+conformance scenarios.
+
 ## License
 
 MIT

package/dist/conformance/audit-tests.d.ts

@@ -0,0 +1,4 @@
+import type { IProviderPlugin } from '@holokai/holo-types/plugin';
+import type { HoloScenario } from './types';
+export declare function registerAuditTests(plugin: IProviderPlugin, scenario: HoloScenario): void;
+//# sourceMappingURL=audit-tests.d.ts.map

package/dist/conformance/audit-tests.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"audit-tests.d.ts","sourceRoot":"","sources":["../../src/conformance/audit-tests.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,4BAA4B,CAAC;AAGhE,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,SAAS,CAAC;AAE1C,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,YAAY,GAAG,IAAI,CAqDxF"}

package/dist/conformance/audit-tests.js

@@ -0,0 +1,55 @@
+import { expect, it } from 'vitest';
+export function registerAuditTests(plugin, scenario) {
+    if (!scenario.audit || !scenario.providerChunks?.length)
+        return;
+    const expectedFailure = scenario.expectedFailure?.audit === true;
+    const t = expectedFailure ? it.fails : it;
+    t('audit', async () => {
+        const auditor = await resolveAuditor(plugin);
+        const lastChunk = scenario.providerChunks[scenario.providerChunks.length - 1];
+        const audit = scenario.audit;
+        const doneEvent = {
+            type: 'done',
+            requestId: 'test-req',
+            seq: scenario.providerChunks.length - 1,
+            message: lastChunk,
+            text: scenario.expectedText ?? '',
+            ts: Date.now(),
+            metrics: {
+                inputTokens: audit.input_tokens ?? 0,
+                outputTokens: audit.output_tokens ?? 0,
+                totalTokens: (audit.input_tokens ?? 0) + (audit.output_tokens ?? 0),
+                timeToFirstToken: 0,
+                totalProcessingTime: 0,
+                startTime: Date.now(),
+            },
+        };
+        const envelope = {
+            source_id: 'test-source-id',
+            request_id: 'test-req',
+            organization_id: 'test-org',
+            provider: { id: 'test-provider-id', name: 'test' },
+            protocol: { id: 'test-protocol-id', name: scenario.protocol, capability: 'chat' },
+            access_model: audit.access_model,
+        };
+        const responseRecord = await auditor.auditResponse(envelope, doneEvent);
+        expect(responseRecord.access_model).toBe(audit.access_model);
+        if (audit.input_tokens !== undefined) {
+            expect(responseRecord.input_tokens).toBe(audit.input_tokens);
+        }
+        if (audit.output_tokens !== undefined) {
+            expect(responseRecord.output_tokens).toBe(audit.output_tokens);
+        }
+        expect(responseRecord.status).toBe(audit.status);
+        if (audit.metadata) {
+            for (const [key, value] of Object.entries(audit.metadata)) {
+                expect(responseRecord.metadata?.[key]).toEqual(value);
+            }
+        }
+    });
+}
+async function resolveAuditor(plugin) {
+    const provider = await plugin.createProvider('test-id', 'test', { apiKey: 'test-key' });
+    return provider.auditor;
+}
+//# sourceMappingURL=audit-tests.js.map

package/dist/conformance/audit-tests.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"audit-tests.js","sourceRoot":"","sources":["../../src/conformance/audit-tests.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAE,EAAE,EAAC,MAAM,QAAQ,CAAC;AAMlC,MAAM,UAAU,kBAAkB,CAAC,MAAuB,EAAE,QAAsB;IAC9E,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE,MAAM;QAAE,OAAO;IAEhE,MAAM,eAAe,GAAG,QAAQ,CAAC,eAAe,EAAE,KAAK,KAAK,IAAI,CAAC;IACjE,MAAM,CAAC,GAAG,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IAE1C,CAAC,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;QAClB,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC,CAAC;QAE7C,MAAM,SAAS,GAAG,QAAQ,CAAC,cAAe,CAAC,QAAQ,CAAC,cAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAChF,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAM,CAAC;QAC9B,MAAM,SAAS,GAAkB;YAC7B,IAAI,EAAE,MAAM;YACZ,SAAS,EAAE,UAAU;YACrB,GAAG,EAAE,QAAQ,CAAC,cAAe,CAAC,MAAM,GAAG,CAAC;YACxC,OAAO,EAAE,SAAS;YAClB,IAAI,EAAE,QAAQ,CAAC,YAAY,IAAI,EAAE;YACjC,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE;YACd,OAAO,EAAE;gBACL,WAAW,EAAE,KAAK,CAAC,YAAY,IAAI,CAAC;gBACpC,YAAY,EAAE,KAAK,CAAC,aAAa,IAAI,CAAC;gBACtC,WAAW,EAAE,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,aAAa,IAAI,CAAC,CAAC;gBACnE,gBAAgB,EAAE,CAAC;gBACnB,mBAAmB,EAAE,CAAC;gBACtB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;aACxB;SACa,CAAC;QAEnB,MAAM,QAAQ,GAA2B;YACrC,SAAS,EAAE,gBAAgB;YAC3B,UAAU,EAAE,UAAU;YACtB,eAAe,EAAE,UAAU;YAC3B,QAAQ,EAAE,EAAC,EAAE,EAAE,kBAAkB,EAAE,IAAI,EAAE,MAAM,EAAQ;YACvD,QAAQ,EAAE,EAAC,EAAE,EAAE,kBAAkB,EAAE,IAAI,EAAE,QAAQ,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAQ;YACtF,YAAY,EAAE,KAAK,CAAC,YAAmB;SAC1C,CAAC;QAEF,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAExE,MAAM,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAC7D,IAAI,KAAK,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;YACnC,MAAM,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QACjE,CAAC;QACD,IAAI,KAAK,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;YACpC,MAAM,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QACnE,CAAC;QACD,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACjD,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACjB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACxD,MAAM,CAAC,cAAc,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC1D,CAAC;QACL,CAAC;IACL,CAAC,CAAC,CAAC;AACP,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,MAAuB;IACjD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,EAAC,MAAM,EAAE,UAAU,EAAC,CAAC,CAAC;IACtF,OAAO,QAAQ,CAAC,OAAO,CAAC;AAC5B,CAAC"}

package/dist/conformance/datastore-tests.d.ts

@@ -0,0 +1,13 @@
+import type { IDatastorePlugin } from '@holokai/holo-types/plugin';
+export interface DatastoreConfig {
+    connectionConfig: Record<string, unknown>;
+    requestTable: string;
+    responseTable: string;
+    costTable: string;
+    cleanup?: (insertedIds: {
+        table: string;
+        id: string;
+    }[]) => Promise<void>;
+}
+export declare function registerDatastoreTests(plugin: IDatastorePlugin, config: DatastoreConfig): void;
+//# sourceMappingURL=datastore-tests.d.ts.map

package/dist/conformance/datastore-tests.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"datastore-tests.d.ts","sourceRoot":"","sources":["../../src/conformance/datastore-tests.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAqB,gBAAgB,EAAC,MAAM,4BAA4B,CAAC;AAErF,MAAM,WAAW,eAAe;IAC5B,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1C,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,CAAC,WAAW,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7E;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,eAAe,GAAG,IAAI,CA6H9F"}

package/dist/conformance/datastore-tests.js

@@ -0,0 +1,121 @@
+import { afterAll, expect, it } from 'vitest';
+import { randomUUID } from 'node:crypto';
+export function registerDatastoreTests(plugin, config) {
+    let instance = null;
+    const insertedIds = [];
+    afterAll(async () => {
+        if (instance?.isConnected()) {
+            await instance.disconnect();
+        }
+        if (config.cleanup && insertedIds.length > 0) {
+            await config.cleanup(insertedIds);
+        }
+    });
+    it('schema is well-formed', () => {
+        const schema = plugin.getSchema?.();
+        expect(schema, 'plugin.getSchema()').toBeDefined();
+        expect(schema.connection, 'schema.connection').toBeDefined();
+        expect(Object.keys(schema.connection.properties ?? {}).length).toBeGreaterThan(0);
+    });
+    it('default mapping is well-formed', () => {
+        const mapping = plugin.getDefaultMapping();
+        expect(mapping, 'plugin.getDefaultMapping()').toBeDefined();
+        const hasSection = mapping.request || mapping.response || mapping.cost;
+        expect(hasSection, 'mapping has at least one section').toBeTruthy();
+        for (const section of ['request', 'response', 'cost']) {
+            const m = mapping[section];
+            if (!m)
+                continue;
+            expect(m.table, `${section}.table`).toBeTruthy();
+            expect(Object.keys(m.fields ?? {}).length, `${section}.fields`).toBeGreaterThan(0);
+        }
+    });
+    it('createInstance + connect', async () => {
+        instance = await plugin.createInstance(config.connectionConfig);
+        await instance.connect();
+        expect(instance.isConnected(), 'isConnected after connect()').toBe(true);
+    });
+    it('testConnection succeeds', async () => {
+        expect(instance, 'instance').not.toBeNull();
+        const result = await instance.testConnection();
+        expect(result.ok, result.error ?? 'testConnection.ok').toBe(true);
+    });
+    it('write request', async () => {
+        expect(instance, 'instance').not.toBeNull();
+        const record = {
+            request_id: randomUUID(),
+            access_model: 'test-model',
+            has_tools: false,
+            created_at: new Date().toISOString(),
+            metadata: JSON.stringify({}),
+        };
+        const result = await instance.write(config.requestTable, record);
+        expect(result.id, 'write request returned id').toBeTruthy();
+        insertedIds.push({ table: config.requestTable, id: result.id });
+    });
+    it('write response', async () => {
+        expect(instance, 'instance').not.toBeNull();
+        const record = {
+            request_id: randomUUID(),
+            access_model: 'test-model',
+            status: 'success',
+            input_tokens: 10,
+            output_tokens: 20,
+            total_tokens: 30,
+            time_to_first_token: 100,
+            total_processing_time: 500,
+            cost: 0,
+            created_at: new Date().toISOString(),
+            metadata: JSON.stringify({}),
+        };
+        const result = await instance.write(config.responseTable, record);
+        expect(result.id, 'write response returned id').toBeTruthy();
+        insertedIds.push({ table: config.responseTable, id: result.id });
+    });
+    it('write cost', async () => {
+        expect(instance, 'instance').not.toBeNull();
+        const responseId = insertedIds.find(r => r.table === config.responseTable)?.id;
+        expect(responseId, 'response id from prior test').toBeTruthy();
+        const record = {
+            response_id: responseId,
+            cost_type: 'provider',
+            input_tokens: 10,
+            output_tokens: 20,
+            cache_read_tokens: 0,
+            cache_write_tokens: 0,
+            input_cost: 0.0001,
+            output_cost: 0.0002,
+            cache_read_cost: 0,
+            cache_write_cost: 0,
+            total_cost: 0.0003,
+            currency: 'USD',
+        };
+        const result = await instance.write(config.costTable, record);
+        if (result.id) {
+            insertedIds.push({ table: config.costTable, id: result.id });
+        }
+    });
+    it.skipIf(false)('updateResponseCost (if supported)', async () => {
+        expect(instance, 'instance').not.toBeNull();
+        if (!instance.updateResponseCost)
+            return;
+        const responseId = insertedIds.find(r => r.table === config.responseTable)?.id;
+        expect(responseId, 'response id from prior test').toBeTruthy();
+        await instance.updateResponseCost(config.responseTable, responseId, 0.0003);
+    });
+    it('rejects unsafe identifiers', async () => {
+        expect(instance, 'instance').not.toBeNull();
+        const unsafe = ["'; DROP TABLE x", 'table name', 'col--umn', 'a;b'];
+        for (const name of unsafe) {
+            await expect(instance.write(name, { id: 'test' }), `should reject "${name}"`).rejects.toThrow();
+        }
+    });
+    it('disconnect', async () => {
+        expect(instance, 'instance').not.toBeNull();
+        if (!instance.isConnected())
+            return; // already torn down by a prior failure
+        await instance.disconnect();
+        expect(instance.isConnected(), 'isConnected after disconnect()').toBe(false);
+    });
+}
+//# sourceMappingURL=datastore-tests.js.map

package/dist/conformance/datastore-tests.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"datastore-tests.js","sourceRoot":"","sources":["../../src/conformance/datastore-tests.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAC,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAC,UAAU,EAAC,MAAM,aAAa,CAAC;AAWvC,MAAM,UAAU,sBAAsB,CAAC,MAAwB,EAAE,MAAuB;IACpF,IAAI,QAAQ,GAA8B,IAAI,CAAC;IAC/C,MAAM,WAAW,GAAoC,EAAE,CAAC;IAExD,QAAQ,CAAC,KAAK,IAAI,EAAE;QAChB,IAAI,QAAQ,EAAE,WAAW,EAAE,EAAE,CAAC;YAC1B,MAAM,QAAQ,CAAC,UAAU,EAAE,CAAC;QAChC,CAAC;QACD,IAAI,MAAM,CAAC,OAAO,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3C,MAAM,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACtC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uBAAuB,EAAE,GAAG,EAAE;QAC7B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,EAAE,CAAC;QACpC,MAAM,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAC,WAAW,EAAE,CAAC;QACnD,MAAM,CAAC,MAAO,CAAC,UAAU,EAAE,mBAAmB,CAAC,CAAC,WAAW,EAAE,CAAC;QAC9D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAO,CAAC,UAAU,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IACvF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;QACtC,MAAM,OAAO,GAAG,MAAM,CAAC,iBAAiB,EAAE,CAAC;QAC3C,MAAM,CAAC,OAAO,EAAE,4BAA4B,CAAC,CAAC,WAAW,EAAE,CAAC;QAC5D,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;QACvE,MAAM,CAAC,UAAU,EAAE,kCAAkC,CAAC,CAAC,UAAU,EAAE,CAAC;QACpE,KAAK,MAAM,OAAO,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,MAAM,CAAU,EAAE,CAAC;YAC7D,MAAM,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;YAC3B,IAAI,CAAC,CAAC;gBAAE,SAAS;YACjB,MAAM,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,OAAO,QAAQ,CAAC,CAAC,UAAU,EAAE,CAAC;YACjD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,GAAG,OAAO,SAAS,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACvF,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0BAA0B,EAAE,KAAK,IAAI,EAAE;QACtC,QAAQ,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAChE,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;QACzB,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,6BAA6B,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;QACrC,MAAM,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG,MAAM,QAAS,CAAC,cAAc,EAAE,CAAC;QAChD,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,KAAK,IAAI,mBAAmB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,eAAe,EAAE,KAAK,IAAI,EAAE;QAC3B,MAAM,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG;YACX,UAAU,EAAE,UAAU,EAAE;YACxB,YAAY,EAAE,YAAY;YAC1B,SAAS,EAAE,KAAK;YAChB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACpC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;SAC/B,CAAC;QACF,MAAM,MAAM,GAAG,MAAM,QAAS,CAAC,KAAK,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;QAClE,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,2BAA2B,CAAC,CAAC,UAAU,EAAE,CAAC;QAC5D,WAAW,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,MAAM,CAAC,YAAY,EAAE,EAAE,EAAE,MAAM,CAAC,EAAG,EAAC,CAAC,CAAC;IACnE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gBAAgB,EAAE,KAAK,IAAI,EAAE;QAC5B,MAAM,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG;YACX,UAAU,EAAE,UAAU,EAAE;YACxB,YAAY,EAAE,YAAY;YAC1B,MAAM,EAAE,SAAS;YACjB,YAAY,EAAE,EAAE;YAChB,aAAa,EAAE,EAAE;YACjB,YAAY,EAAE,EAAE;YAChB,mBAAmB,EAAE,GAAG;YACxB,qBAAqB,EAAE,GAAG;YAC1B,IAAI,EAAE,CAAC;YACP,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACpC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;SAC/B,CAAC;QACF,MAAM,MAAM,GAAG,MAAM,QAAS,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;QACnE,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,4BAA4B,CAAC,CAAC,UAAU,EAAE,CAAC;QAC7D,WAAW,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,MAAM,CAAC,aAAa,EAAE,EAAE,EAAE,MAAM,CAAC,EAAG,EAAC,CAAC,CAAC;IACpE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,YAAY,EAAE,KAAK,IAAI,EAAE;QACxB,MAAM,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC5C,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,MAAM,CAAC,aAAa,CAAC,EAAE,EAAE,CAAC;QAC/E,MAAM,CAAC,UAAU,EAAE,6BAA6B,CAAC,CAAC,UAAU,EAAE,CAAC;QAC/D,MAAM,MAAM,GAAG;YACX,WAAW,EAAE,UAAU;YACvB,SAAS,EAAE,UAAU;YACrB,YAAY,EAAE,EAAE;YAChB,aAAa,EAAE,EAAE;YACjB,iBAAiB,EAAE,CAAC;YACpB,kBAAkB,EAAE,CAAC;YACrB,UAAU,EAAE,MAAM;YAClB,WAAW,EAAE,MAAM;YACnB,eAAe,EAAE,CAAC;YAClB,gBAAgB,EAAE,CAAC;YACnB,UAAU,EAAE,MAAM;YAClB,QAAQ,EAAE,KAAK;SAClB,CAAC;QACF,MAAM,MAAM,GAAG,MAAM,QAAS,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC/D,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;YACZ,WAAW,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAC,CAAC,CAAC;QAC/D,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;QAC7D,MAAM,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC5C,IAAI,CAAC,QAAS,CAAC,kBAAkB;YAAE,OAAO;QAC1C,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,MAAM,CAAC,aAAa,CAAC,EAAE,EAAE,CAAC;QAC/E,MAAM,CAAC,UAAU,EAAE,6BAA6B,CAAC,CAAC,UAAU,EAAE,CAAC;QAC/D,MAAM,QAAS,CAAC,kBAAkB,CAAC,MAAM,CAAC,aAAa,EAAE,UAAW,EAAE,MAAM,CAAC,CAAC;IAClF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,KAAK,IAAI,EAAE;QACxC,MAAM,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG,CAAC,iBAAiB,EAAE,YAAY,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;QACpE,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YACxB,MAAM,MAAM,CAAC,QAAS,CAAC,KAAK,CAAC,IAAI,EAAE,EAAC,EAAE,EAAE,MAAM,EAAC,CAAC,EAAE,kBAAkB,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QACnG,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,YAAY,EAAE,KAAK,IAAI,EAAE;QACxB,MAAM,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC5C,IAAI,CAAC,QAAS,CAAC,WAAW,EAAE;YAAE,OAAO,CAAC,uCAAuC;QAC7E,MAAM,QAAS,CAAC,UAAU,EAAE,CAAC;QAC7B,MAAM,CAAC,QAAS,CAAC,WAAW,EAAE,EAAE,gCAAgC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClF,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/conformance/holo-fixture-loader.d.ts

@@ -0,0 +1,4 @@
+import type { HoloFixture, PluginHoloScenario } from './holo-fixture-types';
+export declare function loadHoloFixtures(): Promise<HoloFixture[]>;
+export declare function loadPluginHoloScenarios(family: string): Promise<PluginHoloScenario[]>;
+//# sourceMappingURL=holo-fixture-loader.d.ts.map

package/dist/conformance/holo-fixture-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"holo-fixture-loader.d.ts","sourceRoot":"","sources":["../../src/conformance/holo-fixture-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,WAAW,EAAmB,kBAAkB,EAAC,MAAM,sBAAsB,CAAC;AAe3F,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAqC/D;AAED,wBAAsB,uBAAuB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAyB3F"}