@databricks/appkit 0.71.0 → 0.73.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +63 -0
- package/NOTICE.md +3 -2
- package/dist/appkit/package.js +1 -1
- package/dist/beta.d.ts +18 -3
- package/dist/beta.js +14 -1
- package/dist/cli/commands/agent/eval.js +120 -0
- package/dist/cli/commands/agent/eval.js.map +1 -0
- package/dist/cli/commands/agent/index.js +18 -0
- package/dist/cli/commands/agent/index.js.map +1 -0
- package/dist/cli/index.js +2 -0
- package/dist/cli/index.js.map +1 -1
- package/dist/connectors/index.js +2 -0
- package/dist/connectors/mlflow/auth.d.ts +28 -0
- package/dist/connectors/mlflow/auth.d.ts.map +1 -0
- package/dist/connectors/mlflow/auth.js +70 -0
- package/dist/connectors/mlflow/auth.js.map +1 -0
- package/dist/connectors/mlflow/client.d.ts +51 -0
- package/dist/connectors/mlflow/client.d.ts.map +1 -0
- package/dist/connectors/mlflow/client.js +93 -0
- package/dist/connectors/mlflow/client.js.map +1 -0
- package/dist/connectors/mlflow/index.d.ts +2 -0
- package/dist/database/errors.js +15 -5
- package/dist/database/errors.js.map +1 -1
- package/dist/database/runtime/data-path.d.ts +7 -0
- package/dist/database/runtime/data-path.d.ts.map +1 -0
- package/dist/database/runtime/data-path.js.map +1 -1
- package/dist/database/runtime/engine/drizzle-data-path.js +7 -5
- package/dist/database/runtime/engine/drizzle-data-path.js.map +1 -1
- package/dist/database/schema-builder/define-schema.d.ts +1 -1
- package/dist/database/schema-builder/define-schema.js +1 -1
- package/dist/database/schema-builder/define-schema.js.map +1 -1
- package/dist/errors/database-validation.d.ts +23 -0
- package/dist/errors/database-validation.d.ts.map +1 -0
- package/dist/errors/database-validation.js +24 -0
- package/dist/errors/database-validation.js.map +1 -0
- package/dist/errors/index.js +1 -0
- package/dist/evals/dataset.d.ts +36 -0
- package/dist/evals/dataset.d.ts.map +1 -0
- package/dist/evals/dataset.js +36 -0
- package/dist/evals/dataset.js.map +1 -0
- package/dist/evals/define-eval.d.ts +26 -0
- package/dist/evals/define-eval.d.ts.map +1 -0
- package/dist/evals/define-eval.js +28 -0
- package/dist/evals/define-eval.js.map +1 -0
- package/dist/evals/discover.d.ts +20 -0
- package/dist/evals/discover.d.ts.map +1 -0
- package/dist/evals/discover.js +49 -0
- package/dist/evals/discover.js.map +1 -0
- package/dist/evals/http-driver.d.ts +33 -0
- package/dist/evals/http-driver.d.ts.map +1 -0
- package/dist/evals/http-driver.js +123 -0
- package/dist/evals/http-driver.js.map +1 -0
- package/dist/evals/index.d.ts +14 -0
- package/dist/evals/index.js +14 -0
- package/dist/evals/judge.d.ts +27 -0
- package/dist/evals/judge.d.ts.map +1 -0
- package/dist/evals/judge.js +77 -0
- package/dist/evals/judge.js.map +1 -0
- package/dist/evals/matchers.d.ts +12 -0
- package/dist/evals/matchers.d.ts.map +1 -0
- package/dist/evals/matchers.js +26 -0
- package/dist/evals/matchers.js.map +1 -0
- package/dist/evals/mlflow-report.d.ts +37 -0
- package/dist/evals/mlflow-report.d.ts.map +1 -0
- package/dist/evals/mlflow-report.js +161 -0
- package/dist/evals/mlflow-report.js.map +1 -0
- package/dist/evals/mlflow-run.d.ts +13 -0
- package/dist/evals/mlflow-run.d.ts.map +1 -0
- package/dist/evals/mlflow-run.js +101 -0
- package/dist/evals/mlflow-run.js.map +1 -0
- package/dist/evals/pool.js +24 -0
- package/dist/evals/pool.js.map +1 -0
- package/dist/evals/report.d.ts +25 -0
- package/dist/evals/report.d.ts.map +1 -0
- package/dist/evals/report.js +57 -0
- package/dist/evals/report.js.map +1 -0
- package/dist/evals/run-eval.d.ts +23 -0
- package/dist/evals/run-eval.d.ts.map +1 -0
- package/dist/evals/run-eval.js +152 -0
- package/dist/evals/run-eval.js.map +1 -0
- package/dist/evals/run-evals.d.ts +94 -0
- package/dist/evals/run-evals.d.ts.map +1 -0
- package/dist/evals/run-evals.js +257 -0
- package/dist/evals/run-evals.js.map +1 -0
- package/dist/evals/types.d.ts +163 -0
- package/dist/evals/types.d.ts.map +1 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/plugin/plugin.d.ts.map +1 -1
- package/dist/plugin/plugin.js +1 -1
- package/dist/plugin/plugin.js.map +1 -1
- package/dist/plugins/agents/agents.js +1 -1
- package/dist/plugins/database/crud/contract.js +17 -8
- package/dist/plugins/database/crud/contract.js.map +1 -1
- package/dist/plugins/database/crud/exposure.js +63 -22
- package/dist/plugins/database/crud/exposure.js.map +1 -1
- package/dist/plugins/database/crud/request.js +50 -0
- package/dist/plugins/database/crud/request.js.map +1 -0
- package/dist/plugins/database/crud/response.js +77 -0
- package/dist/plugins/database/crud/response.js.map +1 -0
- package/dist/plugins/database/crud/routes.js +71 -52
- package/dist/plugins/database/crud/routes.js.map +1 -1
- package/dist/plugins/database/database.d.ts +6 -4
- package/dist/plugins/database/database.d.ts.map +1 -1
- package/dist/plugins/database/database.js +46 -16
- package/dist/plugins/database/database.js.map +1 -1
- package/dist/plugins/database/defaults.js +5 -1
- package/dist/plugins/database/defaults.js.map +1 -1
- package/dist/plugins/database/entity-client.js +143 -10
- package/dist/plugins/database/entity-client.js.map +1 -1
- package/dist/plugins/database/entity-types.d.ts +1 -1
- package/dist/plugins/database/hooks.d.ts +38 -0
- package/dist/plugins/database/hooks.d.ts.map +1 -0
- package/dist/plugins/database/index.d.ts +3 -2
- package/dist/plugins/database/lifecycle.js +67 -28
- package/dist/plugins/database/lifecycle.js.map +1 -1
- package/dist/plugins/database/scope.js +58 -0
- package/dist/plugins/database/scope.js.map +1 -0
- package/dist/plugins/database/types.d.ts +40 -12
- package/dist/plugins/database/types.d.ts.map +1 -1
- package/docs/api/appkit/Class.AppKitError.md +1 -0
- package/docs/api/appkit/Class.DatabaseValidationError.md +191 -0
- package/docs/api/appkit/Class.MlflowClient.md +103 -0
- package/docs/api/appkit/Function.buildAssessments.md +16 -0
- package/docs/api/appkit/Function.configureJudge.md +18 -0
- package/docs/api/appkit/Function.createHttpDriver.md +18 -0
- package/docs/api/appkit/Function.defineEval.md +35 -0
- package/docs/api/appkit/Function.defineSchema.md +1 -1
- package/docs/api/appkit/Function.discoverEvalFiles.md +18 -0
- package/docs/api/appkit/Function.equals.md +18 -0
- package/docs/api/appkit/Function.evalGlyph.md +18 -0
- package/docs/api/appkit/Function.formatEvalDetail.md +18 -0
- package/docs/api/appkit/Function.formatEvalHeadline.md +18 -0
- package/docs/api/appkit/Function.formatEvalResults.md +18 -0
- package/docs/api/appkit/Function.formatSummaryLine.md +18 -0
- package/docs/api/appkit/Function.includes.md +18 -0
- package/docs/api/appkit/Function.isJudgeConfigured.md +10 -0
- package/docs/api/appkit/Function.matches.md +18 -0
- package/docs/api/appkit/Function.normalizeHost.md +18 -0
- package/docs/api/appkit/Function.readEvalDataset.md +21 -0
- package/docs/api/appkit/Function.reportToMlflow.md +23 -0
- package/docs/api/appkit/Function.resolveDatabricksAuth.md +16 -0
- package/docs/api/appkit/Function.resolveWorkspaceClient.md +18 -0
- package/docs/api/appkit/Function.runEval.md +19 -0
- package/docs/api/appkit/Function.runEvalsInDir.md +18 -0
- package/docs/api/appkit/Function.summarize.md +16 -0
- package/docs/api/appkit/Interface.AssertionHandle.md +54 -0
- package/docs/api/appkit/Interface.AssertionResult.md +48 -0
- package/docs/api/appkit/Interface.Assessment.md +83 -0
- package/docs/api/appkit/Interface.CustomJudgeSpec.md +30 -0
- package/docs/api/appkit/Interface.DatabaseValidationIssue.md +21 -0
- package/docs/api/appkit/Interface.DatabricksAuth.md +21 -0
- package/docs/api/appkit/Interface.DatasetRow.md +21 -0
- package/docs/api/appkit/Interface.DiscoveredEval.md +36 -0
- package/docs/api/appkit/Interface.DriveResult.md +58 -0
- package/docs/api/appkit/Interface.EntityMutationHooks.md +173 -0
- package/docs/api/appkit/Interface.EvalDefinition.md +74 -0
- package/docs/api/appkit/Interface.EvalDriver.md +37 -0
- package/docs/api/appkit/Interface.EvalResult.md +83 -0
- package/docs/api/appkit/Interface.EvalRunSummary.md +46 -0
- package/docs/api/appkit/Interface.EvalSummary.md +48 -0
- package/docs/api/appkit/Interface.HookApp.md +12 -0
- package/docs/api/appkit/Interface.HookContext.md +21 -0
- package/docs/api/appkit/Interface.HttpDriverOptions.md +67 -0
- package/docs/api/appkit/Interface.JudgeConfig.md +34 -0
- package/docs/api/appkit/Interface.JudgeScore.md +21 -0
- package/docs/api/appkit/Interface.MatchResult.md +34 -0
- package/docs/api/appkit/Interface.PostResult.md +30 -0
- package/docs/api/appkit/Interface.ReadEvalDatasetOptions.md +34 -0
- package/docs/api/appkit/Interface.ReadSerializerContext.md +21 -0
- package/docs/api/appkit/Interface.ReportOutcome.md +53 -0
- package/docs/api/appkit/Interface.ResolveDatabricksAuthOptions.md +34 -0
- package/docs/api/appkit/Interface.RunEvalOptions.md +45 -0
- package/docs/api/appkit/Interface.RunEvalsOptions.md +214 -0
- package/docs/api/appkit/Interface.TestContext.md +245 -0
- package/docs/api/appkit/TypeAlias.DatabaseApiConfig.md +53 -0
- package/docs/api/appkit/TypeAlias.DatabaseApiWriteOperation.md +8 -0
- package/docs/api/appkit/TypeAlias.DatabaseApiWritesConfig.md +49 -0
- package/docs/api/appkit/TypeAlias.DatabaseExports.md +3 -3
- package/docs/api/appkit/TypeAlias.EntityHooks.md +25 -0
- package/docs/api/appkit/TypeAlias.EvalProgress.md +26 -0
- package/docs/api/appkit/TypeAlias.IDatabaseConfig.md +16 -5
- package/docs/api/appkit/TypeAlias.Matcher.md +18 -0
- package/docs/api/appkit/TypeAlias.ReadSerializer.md +19 -0
- package/docs/api/appkit/TypeAlias.Severity.md +8 -0
- package/docs/api/appkit/TypeAlias.TransactionClient.md +19 -0
- package/docs/api/appkit.md +157 -95
- package/docs/plugins/database.md +144 -0
- package/llms.txt +63 -0
- package/package.json +3 -2
- package/sbom.cdx.json +1 -1
|
@@ -1,22 +1,36 @@
|
|
|
1
1
|
import { Schema } from "../../database/schema-builder/types.js";
|
|
2
|
+
import { EntityMutationHooks } from "./hooks.js";
|
|
2
3
|
|
|
3
4
|
//#region src/plugins/database/types.d.ts
|
|
4
5
|
/** Table names declared by one finalized schema. */
|
|
5
6
|
type SchemaTableName<TSchema extends Schema> = Extract<keyof TSchema["$tables"], string>;
|
|
7
|
+
/** Generated HTTP write operations. */
|
|
8
|
+
type DatabaseApiWriteOperation = "create" | "update" | "delete";
|
|
9
|
+
/** All writes by default; false keeps reads only, and an object narrows writes. */
|
|
10
|
+
type DatabaseApiWritesConfig<TSchema extends Schema> = boolean | {
|
|
11
|
+
/** Tables that remain writable. Defaults to every exposed table. */readonly tables?: readonly SchemaTableName<TSchema>[]; /** Operations that remain enabled. Defaults to create, update, and delete. */
|
|
12
|
+
readonly operations?: readonly DatabaseApiWriteOperation[];
|
|
13
|
+
};
|
|
6
14
|
/**
|
|
7
|
-
*
|
|
15
|
+
* Full generated CRUD for every declared table by default. Set false to disable
|
|
16
|
+
* all generated routes, or use an object to restrict tables and writes.
|
|
17
|
+
* Keyed routes require a public primary key; upsert stays programmatic.
|
|
18
|
+
* Route names must start with a letter, contain only letters, digits, `_`, or
|
|
19
|
+
* `-`, be at most 64 characters, and be unique ignoring case. Invalid names
|
|
20
|
+
* fail setup; exclude internal tables with `api.tables` or use `api: false`.
|
|
8
21
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
* cannot be included
|
|
22
|
+
* Routes run as the app's service principal and apply no per-user filter, so
|
|
23
|
+
* anyone the app admits receives every enabled operation. An exposed table
|
|
24
|
+
* also becomes includable from its neighbours; a relation whose target
|
|
25
|
+
* stays off cannot be included.
|
|
13
26
|
*
|
|
14
27
|
* Text filters accept caller-supplied `like`/`ilike` patterns; a server-side
|
|
15
28
|
* `statement_timeout` cancels a pattern that would otherwise hold its pooled
|
|
16
29
|
* connection to completion.
|
|
17
30
|
*/
|
|
18
|
-
type
|
|
19
|
-
readonly tables
|
|
31
|
+
type DatabaseApiConfig<TSchema extends Schema> = boolean | {
|
|
32
|
+
/** Tables that remain exposed. Defaults to every declared table. */readonly tables?: readonly SchemaTableName<TSchema>[]; /** All writes by default. Set false for read-only routes. */
|
|
33
|
+
readonly writes?: DatabaseApiWritesConfig<TSchema>;
|
|
20
34
|
};
|
|
21
35
|
/** Which entity and generated operation produced the row being shaped. */
|
|
22
36
|
interface ReadSerializerContext {
|
|
@@ -29,14 +43,28 @@ interface ReadSerializerContext {
|
|
|
29
43
|
* serializers run inside the response path and must not add latency there.
|
|
30
44
|
*/
|
|
31
45
|
type ReadSerializer = (row: Record<string, unknown>, context: ReadSerializerContext) => Record<string, unknown>;
|
|
46
|
+
/** Response shaping and mutation lifecycle declared for one table. */
|
|
47
|
+
type EntityHooks<TTable extends string = string> = EntityMutationHooks<TTable> & {
|
|
48
|
+
readonly serialize?: ReadSerializer;
|
|
49
|
+
};
|
|
32
50
|
/** Configuration for one schema-bound DatabasePlugin instance. */
|
|
33
51
|
type IDatabaseConfig<TSchema extends Schema> = {
|
|
34
52
|
readonly schema: TSchema;
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
53
|
+
/**
|
|
54
|
+
* Generated HTTP CRUD is enabled for all tables by default, using the app's
|
|
55
|
+
* service principal. Every admitted caller receives the enabled operations;
|
|
56
|
+
* no per-user or per-row authorization is applied. This plugin does not
|
|
57
|
+
* support OBO. Use custom routes for application-specific authorization.
|
|
58
|
+
*
|
|
59
|
+
* Set `false` to disable routes, `{ writes: false }` for reads only, or
|
|
60
|
+
* `{ tables: ["notes"] }` to expose only selected tables. To disable delete,
|
|
61
|
+
* use `{ writes: { operations: ["create", "update"] } }`.
|
|
62
|
+
*
|
|
63
|
+
* @defaultValue true
|
|
64
|
+
*/
|
|
65
|
+
readonly api?: DatabaseApiConfig<TSchema>;
|
|
66
|
+
readonly hooks?: { readonly [TTable in SchemaTableName<TSchema>]?: EntityHooks<TTable> };
|
|
39
67
|
};
|
|
40
68
|
//#endregion
|
|
41
|
-
export { IDatabaseConfig };
|
|
69
|
+
export { DatabaseApiConfig, DatabaseApiWriteOperation, DatabaseApiWritesConfig, EntityHooks, IDatabaseConfig, ReadSerializer, ReadSerializerContext };
|
|
42
70
|
//# sourceMappingURL=types.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","names":[],"sources":["../../../src/plugins/database/types.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"types.d.ts","names":[],"sources":["../../../src/plugins/database/types.ts"],"mappings":";;;;;KAIY,eAAA,iBAAgC,MAAA,IAAU,OAAA,OAC9C,OAAA;AADR;AAAA,KAMY,yBAAA;;KAGA,uBAAA,iBAAwC,MAAA;EAR5C,6EAYO,MAAA,YAAkB,eAAA,CAAgB,OAAA,KAbY;EAAA,SAe9C,UAAA,YAAsB,yBAAA;AAAA;;;;;;AATrC;;;;;AAGA;;;;;;;KA0BY,iBAAA,iBAAkC,MAAA;EApBgB,6EAwB/C,MAAA,YAAkB,eAAA,CAAgB,OAAA,KA9BG;EAAA,SAgCrC,MAAA,GAAS,uBAAA,CAAwB,OAAA;AAAA;;UAI/B,qBAAA;EAAA,SACN,MAAA;EAAA,SACA,SAAA;AAAA;AAZX;;;;;AAAA,KAoBY,cAAA,IACV,GAAA,EAAK,MAAA,mBACL,OAAA,EAAS,qBAAA,KACN,MAAA;;KAGO,WAAA,mCACV,mBAAA,CAAoB,MAAA;EAAA,SACT,SAAA,GAAY,cAAA;AAAA;;KAOb,eAAA,iBAAgC,MAAA;EAAA,SACjC,MAAA,EAAQ,OAAA;EA9BJ;;;;;AAIf;;;;;AAUA;;EAde,SA2CJ,GAAA,GAAM,iBAAA,CAAkB,OAAA;EAAA,SACxB,KAAA,yBACa,eAAA,CAAgB,OAAA,KAAY,WAAA,CAAY,MAAA;AAAA"}
|
|
@@ -30,6 +30,7 @@ console.error(error.toJSON()); // Safe for logging, sensitive values redacted
|
|
|
30
30
|
* [`AuthenticationError`](./docs/api/appkit/Class.AuthenticationError.md)
|
|
31
31
|
* [`ConfigurationError`](./docs/api/appkit/Class.ConfigurationError.md)
|
|
32
32
|
* [`ConnectionError`](./docs/api/appkit/Class.ConnectionError.md)
|
|
33
|
+
* [`DatabaseValidationError`](./docs/api/appkit/Class.DatabaseValidationError.md)
|
|
33
34
|
* [`ExecutionError`](./docs/api/appkit/Class.ExecutionError.md)
|
|
34
35
|
* [`InitializationError`](./docs/api/appkit/Class.InitializationError.md)
|
|
35
36
|
* [`ServerError`](./docs/api/appkit/Class.ServerError.md)
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
# Class: DatabaseValidationError
|
|
2
|
+
|
|
3
|
+
Deliberate validation failure raised by a database mutation hook. Generated routes answer `422` and echo only the issues naming a public column; every other failure raised inside a hook stays an opaque server error.
|
|
4
|
+
|
|
5
|
+
## Extends[](#extends "Direct link to Extends")
|
|
6
|
+
|
|
7
|
+
* [`AppKitError`](./docs/api/appkit/Class.AppKitError.md)
|
|
8
|
+
|
|
9
|
+
## Constructors[](#constructors "Direct link to Constructors")
|
|
10
|
+
|
|
11
|
+
### Constructor[](#constructor "Direct link to Constructor")
|
|
12
|
+
|
|
13
|
+
```ts
|
|
14
|
+
new DatabaseValidationError(message: string, issues: readonly DatabaseValidationIssue[]): DatabaseValidationError;
|
|
15
|
+
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
#### Parameters[](#parameters "Direct link to Parameters")
|
|
19
|
+
|
|
20
|
+
| Parameter | Type | Default value |
|
|
21
|
+
| --------- | ----------------------------------------------------------------------------------------------------- | ------------- |
|
|
22
|
+
| `message` | `string` | `undefined` |
|
|
23
|
+
| `issues` | readonly [`DatabaseValidationIssue`](./docs/api/appkit/Interface.DatabaseValidationIssue.md)\[] | `[]` |
|
|
24
|
+
|
|
25
|
+
#### Returns[](#returns "Direct link to Returns")
|
|
26
|
+
|
|
27
|
+
`DatabaseValidationError`
|
|
28
|
+
|
|
29
|
+
#### Overrides[](#overrides "Direct link to Overrides")
|
|
30
|
+
|
|
31
|
+
[`AppKitError`](./docs/api/appkit/Class.AppKitError.md).[`constructor`](./docs/api/appkit/Class.AppKitError.md#constructor)
|
|
32
|
+
|
|
33
|
+
## Properties[](#properties "Direct link to Properties")
|
|
34
|
+
|
|
35
|
+
### \_clientMessage?[](#_clientmessage "Direct link to _clientMessage?")
|
|
36
|
+
|
|
37
|
+
```ts
|
|
38
|
+
protected readonly optional _clientMessage: string;
|
|
39
|
+
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Client-safe error message. When set, callers serializing the error to a client (SSE, HTTP body) MUST prefer `clientMessage` over `message` — `message` may contain raw upstream / SDK text including statement fragments, internal object names, and correlation IDs.
|
|
43
|
+
|
|
44
|
+
Subclasses can set this in their constructor for a fixed sanitized string. When unset, `clientMessage` defaults to a generic per-code string (see the getter), and the raw `message` is kept server-side only.
|
|
45
|
+
|
|
46
|
+
#### Inherited from[](#inherited-from "Direct link to Inherited from")
|
|
47
|
+
|
|
48
|
+
[`AppKitError`](./docs/api/appkit/Class.AppKitError.md).[`_clientMessage`](./docs/api/appkit/Class.AppKitError.md#_clientmessage)
|
|
49
|
+
|
|
50
|
+
***
|
|
51
|
+
|
|
52
|
+
### cause?[](#cause "Direct link to cause?")
|
|
53
|
+
|
|
54
|
+
```ts
|
|
55
|
+
readonly optional cause: Error;
|
|
56
|
+
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Optional cause of the error
|
|
60
|
+
|
|
61
|
+
#### Inherited from[](#inherited-from-1 "Direct link to Inherited from")
|
|
62
|
+
|
|
63
|
+
[`AppKitError`](./docs/api/appkit/Class.AppKitError.md).[`cause`](./docs/api/appkit/Class.AppKitError.md#cause)
|
|
64
|
+
|
|
65
|
+
***
|
|
66
|
+
|
|
67
|
+
### code[](#code "Direct link to code")
|
|
68
|
+
|
|
69
|
+
```ts
|
|
70
|
+
readonly code: "DATABASE_VALIDATION_ERROR" = "DATABASE_VALIDATION_ERROR";
|
|
71
|
+
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Error code for programmatic error handling
|
|
75
|
+
|
|
76
|
+
#### Overrides[](#overrides-1 "Direct link to Overrides")
|
|
77
|
+
|
|
78
|
+
[`AppKitError`](./docs/api/appkit/Class.AppKitError.md).[`code`](./docs/api/appkit/Class.AppKitError.md#code)
|
|
79
|
+
|
|
80
|
+
***
|
|
81
|
+
|
|
82
|
+
### context?[](#context "Direct link to context?")
|
|
83
|
+
|
|
84
|
+
```ts
|
|
85
|
+
readonly optional context: Record<string, unknown>;
|
|
86
|
+
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
Additional context for the error
|
|
90
|
+
|
|
91
|
+
#### Inherited from[](#inherited-from-2 "Direct link to Inherited from")
|
|
92
|
+
|
|
93
|
+
[`AppKitError`](./docs/api/appkit/Class.AppKitError.md).[`context`](./docs/api/appkit/Class.AppKitError.md#context)
|
|
94
|
+
|
|
95
|
+
***
|
|
96
|
+
|
|
97
|
+
### isRetryable[](#isretryable "Direct link to isRetryable")
|
|
98
|
+
|
|
99
|
+
```ts
|
|
100
|
+
readonly isRetryable: false = false;
|
|
101
|
+
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Whether this error type is generally safe to retry
|
|
105
|
+
|
|
106
|
+
#### Overrides[](#overrides-2 "Direct link to Overrides")
|
|
107
|
+
|
|
108
|
+
[`AppKitError`](./docs/api/appkit/Class.AppKitError.md).[`isRetryable`](./docs/api/appkit/Class.AppKitError.md#isretryable)
|
|
109
|
+
|
|
110
|
+
***
|
|
111
|
+
|
|
112
|
+
### issues[](#issues "Direct link to issues")
|
|
113
|
+
|
|
114
|
+
```ts
|
|
115
|
+
readonly issues: readonly DatabaseValidationIssue[];
|
|
116
|
+
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
***
|
|
120
|
+
|
|
121
|
+
### statusCode[](#statuscode "Direct link to statusCode")
|
|
122
|
+
|
|
123
|
+
```ts
|
|
124
|
+
readonly statusCode: 422 = 422;
|
|
125
|
+
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
HTTP status code suggestion (can be overridden)
|
|
129
|
+
|
|
130
|
+
#### Overrides[](#overrides-3 "Direct link to Overrides")
|
|
131
|
+
|
|
132
|
+
[`AppKitError`](./docs/api/appkit/Class.AppKitError.md).[`statusCode`](./docs/api/appkit/Class.AppKitError.md#statuscode)
|
|
133
|
+
|
|
134
|
+
## Accessors[](#accessors "Direct link to Accessors")
|
|
135
|
+
|
|
136
|
+
### clientMessage[](#clientmessage "Direct link to clientMessage")
|
|
137
|
+
|
|
138
|
+
#### Get Signature[](#get-signature "Direct link to Get Signature")
|
|
139
|
+
|
|
140
|
+
```ts
|
|
141
|
+
get clientMessage(): string;
|
|
142
|
+
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
Sanitized message safe to forward to clients. Override in subclasses if a more specific default is appropriate.
|
|
146
|
+
|
|
147
|
+
##### Returns[](#returns-1 "Direct link to Returns")
|
|
148
|
+
|
|
149
|
+
`string`
|
|
150
|
+
|
|
151
|
+
#### Inherited from[](#inherited-from-3 "Direct link to Inherited from")
|
|
152
|
+
|
|
153
|
+
[`AppKitError`](./docs/api/appkit/Class.AppKitError.md).[`clientMessage`](./docs/api/appkit/Class.AppKitError.md#clientmessage)
|
|
154
|
+
|
|
155
|
+
## Methods[](#methods "Direct link to Methods")
|
|
156
|
+
|
|
157
|
+
### toJSON()[](#tojson "Direct link to toJSON()")
|
|
158
|
+
|
|
159
|
+
```ts
|
|
160
|
+
toJSON(): Record<string, unknown>;
|
|
161
|
+
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
Convert error to JSON for logging/serialization. Sensitive values in context are automatically redacted.
|
|
165
|
+
|
|
166
|
+
#### Returns[](#returns-2 "Direct link to Returns")
|
|
167
|
+
|
|
168
|
+
`Record`<`string`, `unknown`>
|
|
169
|
+
|
|
170
|
+
#### Inherited from[](#inherited-from-4 "Direct link to Inherited from")
|
|
171
|
+
|
|
172
|
+
[`AppKitError`](./docs/api/appkit/Class.AppKitError.md).[`toJSON`](./docs/api/appkit/Class.AppKitError.md#tojson)
|
|
173
|
+
|
|
174
|
+
***
|
|
175
|
+
|
|
176
|
+
### toString()[](#tostring "Direct link to toString()")
|
|
177
|
+
|
|
178
|
+
```ts
|
|
179
|
+
toString(): string;
|
|
180
|
+
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
Create a human-readable string representation
|
|
184
|
+
|
|
185
|
+
#### Returns[](#returns-3 "Direct link to Returns")
|
|
186
|
+
|
|
187
|
+
`string`
|
|
188
|
+
|
|
189
|
+
#### Inherited from[](#inherited-from-5 "Direct link to Inherited from")
|
|
190
|
+
|
|
191
|
+
[`AppKitError`](./docs/api/appkit/Class.AppKitError.md).[`toString`](./docs/api/appkit/Class.AppKitError.md#tostring)
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# Class: MlflowClient
|
|
2
|
+
|
|
3
|
+
A thin client over the Databricks workspace REST API, owning the host + bearer token so callers (eval-run creation, assessment writes, the judge's serving endpoint) don't each re-derive URLs or re-attach auth. The host is normalized once at construction.
|
|
4
|
+
|
|
5
|
+
## Constructors[](#constructors "Direct link to Constructors")
|
|
6
|
+
|
|
7
|
+
### Constructor[](#constructor "Direct link to Constructor")
|
|
8
|
+
|
|
9
|
+
```ts
|
|
10
|
+
new MlflowClient(host: string, token: string): MlflowClient;
|
|
11
|
+
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
#### Parameters[](#parameters "Direct link to Parameters")
|
|
15
|
+
|
|
16
|
+
| Parameter | Type |
|
|
17
|
+
| --------- | -------- |
|
|
18
|
+
| `host` | `string` |
|
|
19
|
+
| `token` | `string` |
|
|
20
|
+
|
|
21
|
+
#### Returns[](#returns "Direct link to Returns")
|
|
22
|
+
|
|
23
|
+
`MlflowClient`
|
|
24
|
+
|
|
25
|
+
## Properties[](#properties "Direct link to Properties")
|
|
26
|
+
|
|
27
|
+
### baseUrl[](#baseurl "Direct link to baseUrl")
|
|
28
|
+
|
|
29
|
+
```ts
|
|
30
|
+
readonly baseUrl: string;
|
|
31
|
+
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Normalized workspace base URL (scheme guaranteed, no trailing slash).
|
|
35
|
+
|
|
36
|
+
## Methods[](#methods "Direct link to Methods")
|
|
37
|
+
|
|
38
|
+
### post()[](#post "Direct link to post()")
|
|
39
|
+
|
|
40
|
+
```ts
|
|
41
|
+
post<T>(path: string, body: unknown): Promise<T>;
|
|
42
|
+
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
POST JSON to an MLflow REST endpoint. Returns the parsed JSON body, or throws with the status + response text so callers can surface a precise error. Use for calls whose failure should abort (e.g. `runs/create`).
|
|
46
|
+
|
|
47
|
+
The thrown message embeds up to 500 chars of the upstream response body to aid debugging. That is fine for the dev-facing eval CLI, but do NOT relay it into an end-user HTTP response if this client is reused in a request handler — the body can carry workspace-internal detail.
|
|
48
|
+
|
|
49
|
+
#### Type Parameters[](#type-parameters "Direct link to Type Parameters")
|
|
50
|
+
|
|
51
|
+
| Type Parameter | Default type |
|
|
52
|
+
| -------------- | ------------ |
|
|
53
|
+
| `T` | `unknown` |
|
|
54
|
+
|
|
55
|
+
#### Parameters[](#parameters-1 "Direct link to Parameters")
|
|
56
|
+
|
|
57
|
+
| Parameter | Type |
|
|
58
|
+
| --------- | --------- |
|
|
59
|
+
| `path` | `string` |
|
|
60
|
+
| `body` | `unknown` |
|
|
61
|
+
|
|
62
|
+
#### Returns[](#returns-1 "Direct link to Returns")
|
|
63
|
+
|
|
64
|
+
`Promise`<`T`>
|
|
65
|
+
|
|
66
|
+
***
|
|
67
|
+
|
|
68
|
+
### postResult()[](#postresult "Direct link to postResult()")
|
|
69
|
+
|
|
70
|
+
```ts
|
|
71
|
+
postResult(path: string, body: unknown): Promise<PostResult>;
|
|
72
|
+
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
POST JSON without throwing: returns `{ ok, status, error }` so best-effort writes (e.g. per-trace assessments) can be collected and reported without aborting the run.
|
|
76
|
+
|
|
77
|
+
`error` embeds up to 500 chars of the upstream body — same caveat as [post](#post): fine to log for the dev CLI, don't relay it to end users.
|
|
78
|
+
|
|
79
|
+
#### Parameters[](#parameters-2 "Direct link to Parameters")
|
|
80
|
+
|
|
81
|
+
| Parameter | Type |
|
|
82
|
+
| --------- | --------- |
|
|
83
|
+
| `path` | `string` |
|
|
84
|
+
| `body` | `unknown` |
|
|
85
|
+
|
|
86
|
+
#### Returns[](#returns-2 "Direct link to Returns")
|
|
87
|
+
|
|
88
|
+
`Promise`<[`PostResult`](./docs/api/appkit/Interface.PostResult.md)>
|
|
89
|
+
|
|
90
|
+
***
|
|
91
|
+
|
|
92
|
+
### servingEndpointsUrl()[](#servingendpointsurl "Direct link to servingEndpointsUrl()")
|
|
93
|
+
|
|
94
|
+
```ts
|
|
95
|
+
servingEndpointsUrl(): string;
|
|
96
|
+
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
OpenAI-compatible base URL for Databricks Model Serving, used as the judge's `OPENAI_BASE_URL`. Same workspace host + token as the MLflow REST calls.
|
|
100
|
+
|
|
101
|
+
#### Returns[](#returns-3 "Direct link to Returns")
|
|
102
|
+
|
|
103
|
+
`string`
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Function: buildAssessments()
|
|
2
|
+
|
|
3
|
+
```ts
|
|
4
|
+
function buildAssessments(result: EvalResult): Assessment[];
|
|
5
|
+
|
|
6
|
+
```
|
|
7
|
+
|
|
8
|
+
## Parameters[](#parameters "Direct link to Parameters")
|
|
9
|
+
|
|
10
|
+
| Parameter | Type |
|
|
11
|
+
| --------- | --------------------------------------------------------------- |
|
|
12
|
+
| `result` | [`EvalResult`](./docs/api/appkit/Interface.EvalResult.md) |
|
|
13
|
+
|
|
14
|
+
## Returns[](#returns "Direct link to Returns")
|
|
15
|
+
|
|
16
|
+
[`Assessment`](./docs/api/appkit/Interface.Assessment.md)\[]
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Function: configureJudge()
|
|
2
|
+
|
|
3
|
+
```ts
|
|
4
|
+
function configureJudge(config: JudgeConfig): Promise<void>;
|
|
5
|
+
|
|
6
|
+
```
|
|
7
|
+
|
|
8
|
+
Configure the judge once. Sets the OpenAI-compatible client env autoevals reads and the default judge model. No-op-safe: on failure, judging stays disabled and [isJudgeConfigured](./docs/api/appkit/Function.isJudgeConfigured.md) returns false.
|
|
9
|
+
|
|
10
|
+
## Parameters[](#parameters "Direct link to Parameters")
|
|
11
|
+
|
|
12
|
+
| Parameter | Type |
|
|
13
|
+
| --------- | ----------------------------------------------------------------- |
|
|
14
|
+
| `config` | [`JudgeConfig`](./docs/api/appkit/Interface.JudgeConfig.md) |
|
|
15
|
+
|
|
16
|
+
## Returns[](#returns "Direct link to Returns")
|
|
17
|
+
|
|
18
|
+
`Promise`<`void`>
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Function: createHttpDriver()
|
|
2
|
+
|
|
3
|
+
```ts
|
|
4
|
+
function createHttpDriver(options: HttpDriverOptions): EvalDriver;
|
|
5
|
+
|
|
6
|
+
```
|
|
7
|
+
|
|
8
|
+
Drives an agent by POSTing to a running app's chat endpoint and parsing the SSE response. Keeps the thread id across `send`s so multi-turn evals share a conversation. Agent/stream errors surface as `succeeded: false` rather than throwing, so `t.succeeded()` can assert on them.
|
|
9
|
+
|
|
10
|
+
## Parameters[](#parameters "Direct link to Parameters")
|
|
11
|
+
|
|
12
|
+
| Parameter | Type |
|
|
13
|
+
| --------- | ----------------------------------------------------------------------------- |
|
|
14
|
+
| `options` | [`HttpDriverOptions`](./docs/api/appkit/Interface.HttpDriverOptions.md) |
|
|
15
|
+
|
|
16
|
+
## Returns[](#returns "Direct link to Returns")
|
|
17
|
+
|
|
18
|
+
[`EvalDriver`](./docs/api/appkit/Interface.EvalDriver.md)
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# Function: defineEval()
|
|
2
|
+
|
|
3
|
+
```ts
|
|
4
|
+
function defineEval(def: EvalDefinition): EvalDefinition;
|
|
5
|
+
|
|
6
|
+
```
|
|
7
|
+
|
|
8
|
+
Define an agent eval. Default-export the result from a `server/agents/<id>/evals/*.eval.ts` file.
|
|
9
|
+
|
|
10
|
+
## Parameters[](#parameters "Direct link to Parameters")
|
|
11
|
+
|
|
12
|
+
| Parameter | Type |
|
|
13
|
+
| --------- | ----------------------------------------------------------------------- |
|
|
14
|
+
| `def` | [`EvalDefinition`](./docs/api/appkit/Interface.EvalDefinition.md) |
|
|
15
|
+
|
|
16
|
+
## Returns[](#returns "Direct link to Returns")
|
|
17
|
+
|
|
18
|
+
[`EvalDefinition`](./docs/api/appkit/Interface.EvalDefinition.md)
|
|
19
|
+
|
|
20
|
+
## Example[](#example "Direct link to Example")
|
|
21
|
+
|
|
22
|
+
```ts
|
|
23
|
+
import { defineEval, includes } from "@databricks/appkit/beta";
|
|
24
|
+
|
|
25
|
+
export default defineEval({
|
|
26
|
+
description: "Weather agent basic coverage",
|
|
27
|
+
async test(t) {
|
|
28
|
+
await t.send("What's the weather in Brooklyn?");
|
|
29
|
+
t.succeeded();
|
|
30
|
+
t.calledTool("get_weather");
|
|
31
|
+
t.check(t.reply, includes("Sunny"));
|
|
32
|
+
},
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
```
|
|
@@ -5,7 +5,7 @@ function defineSchema<TTables>(builder: (context: SchemaBuilderContext) => TTabl
|
|
|
5
5
|
|
|
6
6
|
```
|
|
7
7
|
|
|
8
|
-
Compile one declared schema. The returned type keeps the table names the builder returned, so `
|
|
8
|
+
Compile one declared schema. The returned type keeps the table names the builder returned, so `api.tables` and `hooks` can name only real tables.
|
|
9
9
|
|
|
10
10
|
## Type Parameters[](#type-parameters "Direct link to Type Parameters")
|
|
11
11
|
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Function: discoverEvalFiles()
|
|
2
|
+
|
|
3
|
+
```ts
|
|
4
|
+
function discoverEvalFiles(rootDir: string): DiscoveredEval[];
|
|
5
|
+
|
|
6
|
+
```
|
|
7
|
+
|
|
8
|
+
Discover evals under `<rootDir>/server/agents/<agent>/evals/` — co-located with each agent's `agent.{md,ts}` (same folder-per-agent layout the agents plugin discovers). The agent id is the folder name; the eval id is the file path relative to that evals dir with `.eval.ts` stripped. Sorted + stable.
|
|
9
|
+
|
|
10
|
+
## Parameters[](#parameters "Direct link to Parameters")
|
|
11
|
+
|
|
12
|
+
| Parameter | Type |
|
|
13
|
+
| --------- | -------- |
|
|
14
|
+
| `rootDir` | `string` |
|
|
15
|
+
|
|
16
|
+
## Returns[](#returns "Direct link to Returns")
|
|
17
|
+
|
|
18
|
+
[`DiscoveredEval`](./docs/api/appkit/Interface.DiscoveredEval.md)\[]
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Function: equals()
|
|
2
|
+
|
|
3
|
+
```ts
|
|
4
|
+
function equals(expected: string): Matcher;
|
|
5
|
+
|
|
6
|
+
```
|
|
7
|
+
|
|
8
|
+
Passes when the value equals `expected` exactly.
|
|
9
|
+
|
|
10
|
+
## Parameters[](#parameters "Direct link to Parameters")
|
|
11
|
+
|
|
12
|
+
| Parameter | Type |
|
|
13
|
+
| ---------- | -------- |
|
|
14
|
+
| `expected` | `string` |
|
|
15
|
+
|
|
16
|
+
## Returns[](#returns "Direct link to Returns")
|
|
17
|
+
|
|
18
|
+
[`Matcher`](./docs/api/appkit/TypeAlias.Matcher.md)
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Function: evalGlyph()
|
|
2
|
+
|
|
3
|
+
```ts
|
|
4
|
+
function evalGlyph(result: EvalResult): string;
|
|
5
|
+
|
|
6
|
+
```
|
|
7
|
+
|
|
8
|
+
Status glyph for a single eval result.
|
|
9
|
+
|
|
10
|
+
## Parameters[](#parameters "Direct link to Parameters")
|
|
11
|
+
|
|
12
|
+
| Parameter | Type |
|
|
13
|
+
| --------- | --------------------------------------------------------------- |
|
|
14
|
+
| `result` | [`EvalResult`](./docs/api/appkit/Interface.EvalResult.md) |
|
|
15
|
+
|
|
16
|
+
## Returns[](#returns "Direct link to Returns")
|
|
17
|
+
|
|
18
|
+
`string`
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Function: formatEvalDetail()
|
|
2
|
+
|
|
3
|
+
```ts
|
|
4
|
+
function formatEvalDetail(result: EvalResult): string[];
|
|
5
|
+
|
|
6
|
+
```
|
|
7
|
+
|
|
8
|
+
Indented detail lines for a failing eval (error + failing assertions).
|
|
9
|
+
|
|
10
|
+
## Parameters[](#parameters "Direct link to Parameters")
|
|
11
|
+
|
|
12
|
+
| Parameter | Type |
|
|
13
|
+
| --------- | --------------------------------------------------------------- |
|
|
14
|
+
| `result` | [`EvalResult`](./docs/api/appkit/Interface.EvalResult.md) |
|
|
15
|
+
|
|
16
|
+
## Returns[](#returns "Direct link to Returns")
|
|
17
|
+
|
|
18
|
+
`string`\[]
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Function: formatEvalHeadline()
|
|
2
|
+
|
|
3
|
+
```ts
|
|
4
|
+
function formatEvalHeadline(result: EvalResult): string;
|
|
5
|
+
|
|
6
|
+
```
|
|
7
|
+
|
|
8
|
+
The one-line header for a single eval result (no failure detail).
|
|
9
|
+
|
|
10
|
+
## Parameters[](#parameters "Direct link to Parameters")
|
|
11
|
+
|
|
12
|
+
| Parameter | Type |
|
|
13
|
+
| --------- | --------------------------------------------------------------- |
|
|
14
|
+
| `result` | [`EvalResult`](./docs/api/appkit/Interface.EvalResult.md) |
|
|
15
|
+
|
|
16
|
+
## Returns[](#returns "Direct link to Returns")
|
|
17
|
+
|
|
18
|
+
`string`
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Function: formatEvalResults()
|
|
2
|
+
|
|
3
|
+
```ts
|
|
4
|
+
function formatEvalResults(results: EvalResult[]): string;
|
|
5
|
+
|
|
6
|
+
```
|
|
7
|
+
|
|
8
|
+
Render all results as a human-readable console report (non-streaming).
|
|
9
|
+
|
|
10
|
+
## Parameters[](#parameters "Direct link to Parameters")
|
|
11
|
+
|
|
12
|
+
| Parameter | Type |
|
|
13
|
+
| --------- | ------------------------------------------------------------------ |
|
|
14
|
+
| `results` | [`EvalResult`](./docs/api/appkit/Interface.EvalResult.md)\[] |
|
|
15
|
+
|
|
16
|
+
## Returns[](#returns "Direct link to Returns")
|
|
17
|
+
|
|
18
|
+
`string`
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Function: formatSummaryLine()
|
|
2
|
+
|
|
3
|
+
```ts
|
|
4
|
+
function formatSummaryLine(results: EvalResult[]): string;
|
|
5
|
+
|
|
6
|
+
```
|
|
7
|
+
|
|
8
|
+
The final PASS/FAIL summary line.
|
|
9
|
+
|
|
10
|
+
## Parameters[](#parameters "Direct link to Parameters")
|
|
11
|
+
|
|
12
|
+
| Parameter | Type |
|
|
13
|
+
| --------- | ------------------------------------------------------------------ |
|
|
14
|
+
| `results` | [`EvalResult`](./docs/api/appkit/Interface.EvalResult.md)\[] |
|
|
15
|
+
|
|
16
|
+
## Returns[](#returns "Direct link to Returns")
|
|
17
|
+
|
|
18
|
+
`string`
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Function: includes()
|
|
2
|
+
|
|
3
|
+
```ts
|
|
4
|
+
function includes(substring: string): Matcher;
|
|
5
|
+
|
|
6
|
+
```
|
|
7
|
+
|
|
8
|
+
Passes when the value contains `substring`.
|
|
9
|
+
|
|
10
|
+
## Parameters[](#parameters "Direct link to Parameters")
|
|
11
|
+
|
|
12
|
+
| Parameter | Type |
|
|
13
|
+
| ----------- | -------- |
|
|
14
|
+
| `substring` | `string` |
|
|
15
|
+
|
|
16
|
+
## Returns[](#returns "Direct link to Returns")
|
|
17
|
+
|
|
18
|
+
[`Matcher`](./docs/api/appkit/TypeAlias.Matcher.md)
|