@fabric-harness/databricks 7.0.1 → 7.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent.d.ts.map +1 -1
- package/dist/agent.js +54 -21
- package/dist/agent.js.map +1 -1
- package/dist/capabilities.d.ts.map +1 -1
- package/dist/certification-environment.d.ts.map +1 -1
- package/dist/certify.js +15 -7
- package/dist/certify.js.map +1 -1
- package/dist/compatibility.d.ts +1 -1
- package/dist/genie-authoring.d.ts +10 -0
- package/dist/genie-authoring.d.ts.map +1 -1
- package/dist/index.d.ts +11 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1848 -30
- package/dist/index.js.map +1 -1
- package/dist/mock-provider.d.ts +1 -0
- package/dist/mock-provider.d.ts.map +1 -1
- package/dist/model.d.ts +7 -0
- package/dist/model.d.ts.map +1 -1
- package/dist/ontology-apply.d.ts +156 -0
- package/dist/ontology-apply.d.ts.map +1 -0
- package/dist/ontology-drift.d.ts +205 -0
- package/dist/ontology-drift.d.ts.map +1 -0
- package/dist/ontology-export.d.ts +181 -0
- package/dist/ontology-export.d.ts.map +1 -0
- package/dist/ontology-feed-pipeline.d.ts +194 -0
- package/dist/ontology-feed-pipeline.d.ts.map +1 -0
- package/dist/ontology-feed.d.ts +49 -0
- package/dist/ontology-feed.d.ts.map +1 -0
- package/dist/ontology-genie-spec.d.ts +31 -0
- package/dist/ontology-genie-spec.d.ts.map +1 -0
- package/dist/ontology-glossary.d.ts +54 -0
- package/dist/ontology-glossary.d.ts.map +1 -0
- package/dist/ontology-metric-view.d.ts +42 -0
- package/dist/ontology-metric-view.d.ts.map +1 -0
- package/package.json +4 -4
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
import type { AssetEventEnvelope } from "@fabricorg/platform/events";
|
|
2
|
+
import type { DatabricksFeedTableSpec } from "./ontology-export.js";
|
|
3
|
+
import { type DatabricksOntologyFeedRecordProducer } from "./ontology-feed.js";
|
|
4
|
+
/**
|
|
5
|
+
* Append/hydration pipeline (P1-2) for the operational-truth feed — the
|
|
6
|
+
* credential-free counterpart of the P0-6 governed apply path. Where
|
|
7
|
+
* `applyModuleOntologyExport` lands the feed table DDL, `planOntologyFeedHydration`
|
|
8
|
+
* turns recorded Platform history (AssetEvent / ActionInvocation /
|
|
9
|
+
* PolicyEvaluation records) into merge-ready rows plus one deterministic
|
|
10
|
+
* `MERGE` statement per feed table. It is pure: no network, no clock, no
|
|
11
|
+
* credential resolution — the caller injects the feed table specs from the
|
|
12
|
+
* plan, the recorded history, and the vertical's redacted-record producer.
|
|
13
|
+
*
|
|
14
|
+
* Contracts (design §3.7, §5, §6 P1-2):
|
|
15
|
+
*
|
|
16
|
+
* - **Replay-safe hydration, idempotent re-run.** Every table merges on its
|
|
17
|
+
* Platform id column (`spec.replayKey` — `event_id`, `invocation_id`,
|
|
18
|
+
* `evaluation_id`) with `MERGE ... WHEN NOT MATCHED THEN INSERT`, so
|
|
19
|
+
* re-hydrating recorded history inserts no duplicate rows. Within one batch,
|
|
20
|
+
* a replayed record (an overlapping hydration window) is deduplicated on the
|
|
21
|
+
* replay key, first occurrence wins, and the drop is counted. Rows are
|
|
22
|
+
* sorted by replay key before rendering, so identical history produces a
|
|
23
|
+
* byte-identical plan and the embedded hydration fingerprint is stable.
|
|
24
|
+
* - **Redaction ownership stays with the vertical (P1-3, KD-10).** Every raw
|
|
25
|
+
* record passes through the caller's producer before allowlist projection;
|
|
26
|
+
* the pipeline fails closed without it (inherited from
|
|
27
|
+
* `materializeOntologyFeedRows`). Harness defines no redaction policy.
|
|
28
|
+
* - **PII/PHI never reaches the merge statement.** Rows are projected onto
|
|
29
|
+
* each spec's `publishableFields` allowlist before rendering, so the emitted
|
|
30
|
+
* SQL carries allowlisted values only.
|
|
31
|
+
* - **Per-tenant SP identity is wiring, not a pipeline input (design §5).**
|
|
32
|
+
* The pipeline never resolves credentials: the scheduled feed pipeline runs
|
|
33
|
+
* as a per-tenant workspace service principal whose client the caller
|
|
34
|
+
* constructs, and the emitted statements execute through the existing
|
|
35
|
+
* governed statement seam (`runStatement` with a
|
|
36
|
+
* `DatabricksStatementClient` built for that principal). Tenant scoping
|
|
37
|
+
* stays a `tenant_id` column enforced by UC grants per consumer.
|
|
38
|
+
* - **Host-runtime orthogonal (KD-11).** History arrives as data; nothing here
|
|
39
|
+
* reads host execution internals.
|
|
40
|
+
*
|
|
41
|
+
* Record shapes: `@fabricorg/platform@0.10.0` ships `AssetEventEnvelope`
|
|
42
|
+
* (`@fabricorg/platform/events`), consumed type-only exactly like the
|
|
43
|
+
* manifest contract — `assetEventRecordFromEnvelope` maps it onto the feed
|
|
44
|
+
* record fields. Platform ships no durable ActionInvocation or
|
|
45
|
+
* PolicyEvaluation record contract, so Harness defines the minimal record
|
|
46
|
+
* types below; the vertical's durable history reader produces them.
|
|
47
|
+
*/
|
|
48
|
+
/**
|
|
49
|
+
* Minimal AssetEvent record the feed pipeline consumes (design §3.7).
|
|
50
|
+
* `event_id` is required — it is the replay key — and any additional raw
|
|
51
|
+
* fields pass through to the vertical's producer untouched (raw payloads,
|
|
52
|
+
* PII-bearing fields) to be redacted or dropped there; the publishable-field
|
|
53
|
+
* allowlist is the last line.
|
|
54
|
+
*/
|
|
55
|
+
export interface DatabricksOntologyAssetEventRecord {
|
|
56
|
+
/** Platform AssetEvent id — the replay key hydration merges on. */
|
|
57
|
+
event_id: string;
|
|
58
|
+
/** Declared event type from the module manifest. */
|
|
59
|
+
event_type?: string | undefined;
|
|
60
|
+
/** Declared event schemaVersion; no upcasting at append time (design Q3). */
|
|
61
|
+
schema_version?: number | undefined;
|
|
62
|
+
/** Fabric module namespace that emitted the event. */
|
|
63
|
+
namespace?: string | undefined;
|
|
64
|
+
/** Tenant scope, carried as a column and enforced by UC grants. */
|
|
65
|
+
tenant_id?: string | undefined;
|
|
66
|
+
object_type?: string | undefined;
|
|
67
|
+
object_id?: string | undefined;
|
|
68
|
+
/** State-machine state after the event, when the module binds one. */
|
|
69
|
+
state?: string | undefined;
|
|
70
|
+
/** Domain time the event occurred (ISO 8601). */
|
|
71
|
+
occurred_at?: string | undefined;
|
|
72
|
+
correlation_id?: string | undefined;
|
|
73
|
+
causation_id?: string | undefined;
|
|
74
|
+
/** Time the event was recorded into durable history (ISO 8601). */
|
|
75
|
+
recorded_at?: string | undefined;
|
|
76
|
+
/** Raw fields the vertical's producer must redact or drop. */
|
|
77
|
+
[field: string]: unknown;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Minimal ActionInvocation record the feed pipeline consumes. Raw parameters
|
|
81
|
+
* travel only to the vertical's producer; the feed table carries
|
|
82
|
+
* `redacted_parameters` only (design §3.7, KD-10).
|
|
83
|
+
*/
|
|
84
|
+
export interface DatabricksOntologyActionInvocationRecord {
|
|
85
|
+
/** Platform ActionInvocation id — the replay key hydration merges on. */
|
|
86
|
+
invocation_id: string;
|
|
87
|
+
action_id?: string | undefined;
|
|
88
|
+
action_version?: number | undefined;
|
|
89
|
+
namespace?: string | undefined;
|
|
90
|
+
tenant_id?: string | undefined;
|
|
91
|
+
terminal_status?: string | undefined;
|
|
92
|
+
failure_classification?: string | undefined;
|
|
93
|
+
started_at?: string | undefined;
|
|
94
|
+
completed_at?: string | undefined;
|
|
95
|
+
duration_ms?: number | undefined;
|
|
96
|
+
correlation_id?: string | undefined;
|
|
97
|
+
/** Post-redaction parameters JSON, produced by the vertical's producer. */
|
|
98
|
+
redacted_parameters?: string | undefined;
|
|
99
|
+
/** Raw fields (e.g. `rawParameters`) the vertical's producer must redact. */
|
|
100
|
+
[field: string]: unknown;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Minimal PolicyEvaluation record the feed pipeline consumes — decision,
|
|
104
|
+
* declared-safe reasons, evaluator identity/version, and failure mode; no
|
|
105
|
+
* facts payload beyond declared-safe fields (design §3.7).
|
|
106
|
+
*/
|
|
107
|
+
export interface DatabricksOntologyPolicyEvaluationRecord {
|
|
108
|
+
/** Platform PolicyEvaluation id — the replay key hydration merges on. */
|
|
109
|
+
evaluation_id: string;
|
|
110
|
+
policy_id?: string | undefined;
|
|
111
|
+
policy_version?: number | undefined;
|
|
112
|
+
namespace?: string | undefined;
|
|
113
|
+
tenant_id?: string | undefined;
|
|
114
|
+
decision?: string | undefined;
|
|
115
|
+
reasons?: string[] | undefined;
|
|
116
|
+
evaluator_identity?: string | undefined;
|
|
117
|
+
evaluator_version?: string | undefined;
|
|
118
|
+
failure_mode?: string | undefined;
|
|
119
|
+
evaluated_at?: string | undefined;
|
|
120
|
+
/** Invocation this evaluation gated; joins to action_invocations. */
|
|
121
|
+
invocation_id?: string | undefined;
|
|
122
|
+
/** Raw fields the vertical's producer must redact or drop. */
|
|
123
|
+
[field: string]: unknown;
|
|
124
|
+
}
|
|
125
|
+
/** Recorded Platform history, grouped by feed table kind. */
|
|
126
|
+
export interface DatabricksOntologyFeedHistory {
|
|
127
|
+
assetEvents?: readonly DatabricksOntologyAssetEventRecord[] | undefined;
|
|
128
|
+
actionInvocations?: readonly DatabricksOntologyActionInvocationRecord[] | undefined;
|
|
129
|
+
policyEvaluations?: readonly DatabricksOntologyPolicyEvaluationRecord[] | undefined;
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Map a Platform `AssetEventEnvelope` (`@fabricorg/platform/events`, consumed
|
|
133
|
+
* type-only) onto the feed's asset-event record fields. The envelope `payload`
|
|
134
|
+
* is the vertical's raw event body: it flows to the producer (which owns
|
|
135
|
+
* redaction, including any upcasting — design Q3) and is dropped by the
|
|
136
|
+
* publishable-field allowlist before anything lands. `namespace` is supplied
|
|
137
|
+
* by the caller because the envelope does not carry the module namespace.
|
|
138
|
+
*/
|
|
139
|
+
export declare function assetEventRecordFromEnvelope(envelope: AssetEventEnvelope, namespace?: string): DatabricksOntologyAssetEventRecord;
|
|
140
|
+
/** Input for feed hydration planning. */
|
|
141
|
+
export interface DatabricksOntologyFeedHydrationInput {
|
|
142
|
+
/** Feed table specs from the export plan (carry the allowlist and replay key). */
|
|
143
|
+
feedTables: readonly DatabricksFeedTableSpec[];
|
|
144
|
+
/** Recorded Platform history, grouped by feed table kind. */
|
|
145
|
+
history: DatabricksOntologyFeedHistory;
|
|
146
|
+
/**
|
|
147
|
+
* Required redacted-record producer owned by the vertical (the
|
|
148
|
+
* fabric-autorefi `redactActionParameters` pattern). Hydration fails closed
|
|
149
|
+
* without it; Harness defines no redaction policy of its own.
|
|
150
|
+
*/
|
|
151
|
+
producer: DatabricksOntologyFeedRecordProducer;
|
|
152
|
+
}
|
|
153
|
+
/** Hydration output for one feed table. */
|
|
154
|
+
export interface DatabricksOntologyFeedTableHydration {
|
|
155
|
+
/** The planned feed table spec (kind, table, replayKey, publishableFields). */
|
|
156
|
+
spec: DatabricksFeedTableSpec;
|
|
157
|
+
/**
|
|
158
|
+
* Merge-ready rows: redacted by the caller's producer, projected onto the
|
|
159
|
+
* publishable-field allowlist, deduplicated on the replay key, and sorted by
|
|
160
|
+
* replay key for deterministic output.
|
|
161
|
+
*/
|
|
162
|
+
rows: Array<Record<string, unknown>>;
|
|
163
|
+
/**
|
|
164
|
+
* Records dropped because an earlier record in the same batch already
|
|
165
|
+
* carried their replay key (overlapping hydration windows).
|
|
166
|
+
*/
|
|
167
|
+
droppedDuplicates: number;
|
|
168
|
+
/**
|
|
169
|
+
* Deterministic `MERGE INTO ... WHEN NOT MATCHED THEN INSERT` statement
|
|
170
|
+
* keyed by the replay key; `null` when there are no rows to append.
|
|
171
|
+
* Re-applying it against a table that already holds the keys inserts
|
|
172
|
+
* nothing — the idempotent re-run contract.
|
|
173
|
+
*/
|
|
174
|
+
statement: string | null;
|
|
175
|
+
/** Fingerprint of the table, replay key, and rows; `null` when there are no rows. */
|
|
176
|
+
statementFingerprint: string | null;
|
|
177
|
+
}
|
|
178
|
+
/** A deterministic feed hydration plan: merge-ready rows and statements per table. */
|
|
179
|
+
export interface DatabricksOntologyFeedHydrationPlan {
|
|
180
|
+
/** Canonical fingerprint of the whole hydration plan. */
|
|
181
|
+
fingerprint: string;
|
|
182
|
+
tables: DatabricksOntologyFeedTableHydration[];
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Plan replay-safe hydration of recorded Platform history into the module's
|
|
186
|
+
* feed tables. Pure and credential-free: identical inputs produce a
|
|
187
|
+
* byte-identical plan, and applying the emitted `MERGE` statements is
|
|
188
|
+
* idempotent by construction (`WHEN NOT MATCHED THEN INSERT` on the replay
|
|
189
|
+
* key). Live execution is the caller's wiring — run each statement through
|
|
190
|
+
* the governed statement seam with a per-tenant service-principal client
|
|
191
|
+
* (design §5), exactly as the P0-6 apply path executes feed DDL.
|
|
192
|
+
*/
|
|
193
|
+
export declare function planOntologyFeedHydration(input: DatabricksOntologyFeedHydrationInput): Promise<DatabricksOntologyFeedHydrationPlan>;
|
|
194
|
+
//# sourceMappingURL=ontology-feed-pipeline.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ontology-feed-pipeline.d.ts","sourceRoot":"","sources":["../src/ontology-feed-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAErE,OAAO,KAAK,EACV,uBAAuB,EAExB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,KAAK,oCAAoC,EAE1C,MAAM,oBAAoB,CAAC;AAE5B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AAEH;;;;;;GAMG;AACH,MAAM,WAAW,kCAAkC;IACjD,mEAAmE;IACnE,QAAQ,EAAE,MAAM,CAAC;IACjB,oDAAoD;IACpD,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,6EAA6E;IAC7E,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,sDAAsD;IACtD,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,mEAAmE;IACnE,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,sEAAsE;IACtE,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,iDAAiD;IACjD,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,mEAAmE;IACnE,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,8DAA8D;IAC9D,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED;;;;GAIG;AACH,MAAM,WAAW,wCAAwC;IACvD,yEAAyE;IACzE,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,eAAe,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5C,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,2EAA2E;IAC3E,mBAAmB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC,6EAA6E;IAC7E,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED;;;;GAIG;AACH,MAAM,WAAW,wCAAwC;IACvD,yEAAyE;IACzE,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IAC/B,kBAAkB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,qEAAqE;IACrE,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,8DAA8D;IAC9D,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED,6DAA6D;AAC7D,MAAM,WAAW,6BAA6B;IAC5C,WAAW,CAAC,EAAE,SAAS,kCAAkC,EAAE,GAAG,SAAS,CAAC;IACxE,iBAAiB,CAAC,EAAE,SAAS,wCAAwC,EAAE,GAAG,SAAS,CAAC;IACpF,iBAAiB,CAAC,EAAE,SAAS,wCAAwC,EAAE,GAAG,SAAS,CAAC;CACrF;AAQD;;;;;;;GAOG;AACH,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,kBAAkB,EAC5B,SAAS,CAAC,EAAE,MAAM,GACjB,kCAAkC,CAepC;AAED,yCAAyC;AACzC,MAAM,WAAW,oCAAoC;IACnD,kFAAkF;IAClF,UAAU,EAAE,SAAS,uBAAuB,EAAE,CAAC;IAC/C,6DAA6D;IAC7D,OAAO,EAAE,6BAA6B,CAAC;IACvC;;;;OAIG;IACH,QAAQ,EAAE,oCAAoC,CAAC;CAChD;AAED,2CAA2C;AAC3C,MAAM,WAAW,oCAAoC;IACnD,+EAA+E;IAC/E,IAAI,EAAE,uBAAuB,CAAC;IAC9B;;;;OAIG;IACH,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IACrC;;;OAGG;IACH,iBAAiB,EAAE,MAAM,CAAC;IAC1B;;;;;OAKG;IACH,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,qFAAqF;IACrF,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED,sFAAsF;AACtF,MAAM,WAAW,mCAAmC;IAClD,yDAAyD;IACzD,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,oCAAoC,EAAE,CAAC;CAChD;AAED;;;;;;;;GAQG;AACH,wBAAsB,yBAAyB,CAC7C,KAAK,EAAE,oCAAoC,GAC1C,OAAO,CAAC,mCAAmC,CAAC,CA+D9C"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { DatabricksFeedTableSpec, DatabricksOntologyFeedTableKind } from "./ontology-export.js";
|
|
2
|
+
/** Input for feed table spec generation: one module namespace plus the export target. */
|
|
3
|
+
export interface DatabricksOntologyFeedSpecInput {
|
|
4
|
+
namespace: string;
|
|
5
|
+
catalog: string;
|
|
6
|
+
schema: string;
|
|
7
|
+
/** Tables to generate; defaults to all three (design §3.7). */
|
|
8
|
+
kinds?: readonly DatabricksOntologyFeedTableKind[] | undefined;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Generate the module's feed table specs with deterministic DDL. The spec's
|
|
12
|
+
* `publishableFields` is the explicit allowlist — every column of the table
|
|
13
|
+
* and nothing more — and `replayKey` names the id column the hydration
|
|
14
|
+
* pipeline merges on. Identical inputs produce byte-identical specs.
|
|
15
|
+
*/
|
|
16
|
+
export declare function generateModuleFeedTableSpecs(input: DatabricksOntologyFeedSpecInput): Promise<DatabricksFeedTableSpec[]>;
|
|
17
|
+
/**
|
|
18
|
+
* The vertical's redacted-record producer (P1-3) — the fabric-autorefi
|
|
19
|
+
* `redactActionParameters` pattern generalized to feed records. The producer
|
|
20
|
+
* receives one raw durable history record (AssetEvent, ActionInvocation, or
|
|
21
|
+
* PolicyEvaluation) and returns the redacted, publishable candidate. Harness
|
|
22
|
+
* defines no redaction policy of its own and never sees the producer's input
|
|
23
|
+
* again; only the producer's output is allowlist-projected.
|
|
24
|
+
*/
|
|
25
|
+
export type DatabricksOntologyFeedRecordProducer = (record: Readonly<Record<string, unknown>>) => Record<string, unknown> | Promise<Record<string, unknown>>;
|
|
26
|
+
/** Input for feed row materialization. */
|
|
27
|
+
export interface DatabricksOntologyFeedMaterializeInput {
|
|
28
|
+
/** Feed table spec from the plan (carries the allowlist and replay key). */
|
|
29
|
+
spec: DatabricksFeedTableSpec;
|
|
30
|
+
/** Raw durable history records, exactly as the vertical read them. */
|
|
31
|
+
records: readonly Readonly<Record<string, unknown>>[];
|
|
32
|
+
/**
|
|
33
|
+
* Required redacted-record producer owned by the vertical. Materialization
|
|
34
|
+
* fails closed without it: raw Platform records cannot be materialized
|
|
35
|
+
* without passing through the caller's redaction boundary.
|
|
36
|
+
*/
|
|
37
|
+
producer: DatabricksOntologyFeedRecordProducer;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Materialize append-ready feed rows. Every raw record passes through the
|
|
41
|
+
* caller's redacted-record producer first; the producer's output is then
|
|
42
|
+
* projected onto the spec's `publishableFields` allowlist, so anything not
|
|
43
|
+
* allowlisted — PII/PHI-bearing fields, raw parameters, secrets — is dropped
|
|
44
|
+
* before the row can be staged for a Delta append. The replay key must
|
|
45
|
+
* survive redaction and projection; a row without it is rejected because it
|
|
46
|
+
* could not merge idempotently.
|
|
47
|
+
*/
|
|
48
|
+
export declare function materializeOntologyFeedRows(input: DatabricksOntologyFeedMaterializeInput): Promise<Array<Record<string, unknown>>>;
|
|
49
|
+
//# sourceMappingURL=ontology-feed.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ontology-feed.d.ts","sourceRoot":"","sources":["../src/ontology-feed.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,uBAAuB,EACvB,+BAA+B,EAChC,MAAM,sBAAsB,CAAC;AAwS9B,yFAAyF;AACzF,MAAM,WAAW,+BAA+B;IAC9C,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,+DAA+D;IAC/D,KAAK,CAAC,EAAE,SAAS,+BAA+B,EAAE,GAAG,SAAS,CAAC;CAChE;AAED;;;;;GAKG;AACH,wBAAsB,4BAA4B,CAChD,KAAK,EAAE,+BAA+B,GACrC,OAAO,CAAC,uBAAuB,EAAE,CAAC,CA2BpC;AAsCD;;;;;;;GAOG;AACH,MAAM,MAAM,oCAAoC,GAAG,CACjD,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,KACtC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AAEhE,0CAA0C;AAC1C,MAAM,WAAW,sCAAsC;IACrD,4EAA4E;IAC5E,IAAI,EAAE,uBAAuB,CAAC;IAC9B,sEAAsE;IACtE,OAAO,EAAE,SAAS,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,CAAC;IACtD;;;;OAIG;IACH,QAAQ,EAAE,oCAAoC,CAAC;CAChD;AAED;;;;;;;;GAQG;AACH,wBAAsB,2BAA2B,CAC/C,KAAK,EAAE,sCAAsC,GAC5C,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CA2BzC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { ModuleManifest } from "@fabricorg/platform/manifest";
|
|
2
|
+
import { type DatabricksGenieAgentSpecV2 } from "./genie-authoring.js";
|
|
3
|
+
import type { DatabricksGlossaryTerm, DatabricksOntologyExportGenieOptions, DatabricksOntologyFeedOptions, DatabricksOntologyPolicyClassification } from "./ontology-export.js";
|
|
4
|
+
/** Input for Genie spec derivation: the assembled plan context plus the genie options. */
|
|
5
|
+
export interface DatabricksOntologyGenieSpecInput {
|
|
6
|
+
/** Bundle manifests in declaration order. */
|
|
7
|
+
manifests: readonly ModuleManifest[];
|
|
8
|
+
catalog: string;
|
|
9
|
+
schema: string;
|
|
10
|
+
classifications: readonly DatabricksOntologyPolicyClassification[];
|
|
11
|
+
/** Glossary terms from the P0-4 generator, in plan order. */
|
|
12
|
+
glossaryTerms: readonly DatabricksGlossaryTerm[];
|
|
13
|
+
genie: DatabricksOntologyExportGenieOptions;
|
|
14
|
+
feed?: DatabricksOntologyFeedOptions | undefined;
|
|
15
|
+
}
|
|
16
|
+
/** Derivation output: the normalized canonical spec plus its plan-time fingerprint. */
|
|
17
|
+
export interface DatabricksOntologyGenieSpecResult {
|
|
18
|
+
spec: DatabricksGenieAgentSpecV2;
|
|
19
|
+
/** Canonical fingerprint for P0-6's `expectedFingerprint` conflict check. */
|
|
20
|
+
fingerprint: string;
|
|
21
|
+
}
|
|
22
|
+
/** True when a parameter name contains a built-in or vertical-declared sensitive fragment. */
|
|
23
|
+
export declare function isSensitiveFieldName(name: string, sensitiveFieldPatterns?: readonly string[]): boolean;
|
|
24
|
+
/**
|
|
25
|
+
* Derive a fresh `DatabricksGenieAgentSpecV2` from the manifest bundle. The
|
|
26
|
+
* returned spec is normalized through the same canonicalization
|
|
27
|
+
* `DatabricksGenieAdmin` applies, and the fingerprint is the plan-time
|
|
28
|
+
* counterpart of the admin's conflict-check fingerprint.
|
|
29
|
+
*/
|
|
30
|
+
export declare function deriveOntologyGenieSpec(input: DatabricksOntologyGenieSpecInput): Promise<DatabricksOntologyGenieSpecResult>;
|
|
31
|
+
//# sourceMappingURL=ontology-genie-spec.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ontology-genie-spec.d.ts","sourceRoot":"","sources":["../src/ontology-genie-spec.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAwB,MAAM,8BAA8B,CAAC;AACzF,OAAO,EACL,KAAK,0BAA0B,EAOhC,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EACV,sBAAsB,EACtB,oCAAoC,EACpC,6BAA6B,EAE7B,sCAAsC,EACvC,MAAM,sBAAsB,CAAC;AAuE9B,0FAA0F;AAC1F,MAAM,WAAW,gCAAgC;IAC/C,6CAA6C;IAC7C,SAAS,EAAE,SAAS,cAAc,EAAE,CAAC;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,SAAS,sCAAsC,EAAE,CAAC;IACnE,6DAA6D;IAC7D,aAAa,EAAE,SAAS,sBAAsB,EAAE,CAAC;IACjD,KAAK,EAAE,oCAAoC,CAAC;IAC5C,IAAI,CAAC,EAAE,6BAA6B,GAAG,SAAS,CAAC;CAClD;AAED,uFAAuF;AACvF,MAAM,WAAW,iCAAiC;IAChD,IAAI,EAAE,0BAA0B,CAAC;IACjC,6EAA6E;IAC7E,WAAW,EAAE,MAAM,CAAC;CACrB;AAOD,8FAA8F;AAC9F,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,MAAM,EACZ,sBAAsB,CAAC,EAAE,SAAS,MAAM,EAAE,GACzC,OAAO,CAQT;AA0LD;;;;;GAKG;AACH,wBAAsB,uBAAuB,CAC3C,KAAK,EAAE,gCAAgC,GACtC,OAAO,CAAC,iCAAiC,CAAC,CA0C5C"}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import type { ModuleManifest } from "@fabricorg/platform/manifest";
|
|
2
|
+
import type { DatabricksDomainAssignment, DatabricksGlossaryTerm, DatabricksOntologyPolicyClassification } from "./ontology-export.js";
|
|
3
|
+
/**
|
|
4
|
+
* Glossary term and domain-assignment generator (P0-4) for the Genie Ontology
|
|
5
|
+
* exporter. Pure and deterministic: no network, no clock, no randomness, and
|
|
6
|
+
* manifest declaration order is preserved end to end, so an identical manifest
|
|
7
|
+
* produces byte-identical terms and assignments.
|
|
8
|
+
*
|
|
9
|
+
* Mapping follows design §3.3. Each module contributes, in order: one term per
|
|
10
|
+
* object type (`displayName` drives the human-facing term name; `idPrefix`
|
|
11
|
+
* and the module `idPrefixes` registry are preserved as metadata for
|
|
12
|
+
* identifier parsing), subject-vocabulary terms (actors, not measures), one
|
|
13
|
+
* term per event type with `schemaVersion` recorded in metadata, one term per
|
|
14
|
+
* action with `mutatesDomain: true` (lifecycle vocabulary, read-side phrasing
|
|
15
|
+
* only — Genie never mutates), one business-rule term per policy the P0-3
|
|
16
|
+
* classifier routed to `glossary-business-rule` (gate/compliance data policies
|
|
17
|
+
* and code evaluators), and one term per state-machine state label. Business
|
|
18
|
+
* rules reference `policyId@policyVersion`; data policies additionally carry a
|
|
19
|
+
* `dataDefinition` summary, while code-evaluator logic is Platform-internal
|
|
20
|
+
* and never exported.
|
|
21
|
+
*
|
|
22
|
+
* Deliberate exclusions (§3.3 — the exporter errs toward exporting less):
|
|
23
|
+
* `adapterSteps` (infrastructure, not vocabulary), code-policy bodies
|
|
24
|
+
* (governance internals), and `requiredRoles` / `requiredPermissions` /
|
|
25
|
+
* `eventPhase` (execution governance, not analytics vocabulary) never produce
|
|
26
|
+
* terms. Threshold/ratio policies classified as metric seeds land on the
|
|
27
|
+
* metric view instead and produce no business-rule term here.
|
|
28
|
+
*
|
|
29
|
+
* Every term upsert payload carries `manifestVersion` metadata (§3.5), plus
|
|
30
|
+
* `schemaVersion` on event-type terms. Domain assignments bind one UC domain
|
|
31
|
+
* per module namespace — the `domain` option overrides the default — with the
|
|
32
|
+
* module description as the domain comment. Payloads are plan data only; the
|
|
33
|
+
* governed publish path lands with P0-6 (glossary/domain UC REST surface
|
|
34
|
+
* maturity is tracked as design Q6).
|
|
35
|
+
*/
|
|
36
|
+
/** Glossary output for one module: terms in declaration order plus the domain binding. */
|
|
37
|
+
export interface DatabricksModuleGlossary {
|
|
38
|
+
terms: DatabricksGlossaryTerm[];
|
|
39
|
+
domainAssignment: DatabricksDomainAssignment;
|
|
40
|
+
}
|
|
41
|
+
/** Input for glossary generation: one manifest, its resolved domain, and its policy classifications. */
|
|
42
|
+
export interface DatabricksOntologyGlossaryInput {
|
|
43
|
+
manifest: ModuleManifest;
|
|
44
|
+
/** UC domain for this module — the `domain` option override or the namespace. */
|
|
45
|
+
domain: string;
|
|
46
|
+
classifications: readonly DatabricksOntologyPolicyClassification[];
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Generate the glossary terms and domain assignment for one module manifest.
|
|
50
|
+
* Deterministic: identical inputs produce byte-identical output, preserving
|
|
51
|
+
* manifest declaration order within each vocabulary group.
|
|
52
|
+
*/
|
|
53
|
+
export declare function generateModuleGlossary(input: DatabricksOntologyGlossaryInput): DatabricksModuleGlossary;
|
|
54
|
+
//# sourceMappingURL=ontology-glossary.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ontology-glossary.d.ts","sourceRoot":"","sources":["../src/ontology-glossary.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAEnE,OAAO,KAAK,EACV,0BAA0B,EAC1B,sBAAsB,EACtB,sCAAsC,EACvC,MAAM,sBAAsB,CAAC;AAE9B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,0FAA0F;AAC1F,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,sBAAsB,EAAE,CAAC;IAChC,gBAAgB,EAAE,0BAA0B,CAAC;CAC9C;AAED,wGAAwG;AACxG,MAAM,WAAW,+BAA+B;IAC9C,QAAQ,EAAE,cAAc,CAAC;IACzB,iFAAiF;IACjF,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,SAAS,sCAAsC,EAAE,CAAC;CACpE;AAsCD;;;;GAIG;AACH,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,+BAA+B,GACrC,wBAAwB,CA0I1B"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import type { ModuleManifest } from "@fabricorg/platform/manifest";
|
|
2
|
+
import type { DatabricksOntologyPolicyClassification } from "./ontology-export.js";
|
|
3
|
+
/** Options accepted by the policy classifier. */
|
|
4
|
+
export interface DatabricksOntologyPolicyClassifierOptions {
|
|
5
|
+
/** Explicit opt-in promoting code-evaluator policies to metric seeds (design §3.4). */
|
|
6
|
+
metricSeedPolicyIds?: readonly string[] | undefined;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Classify every policy a manifest declares (design §3.4). Declaration order
|
|
10
|
+
* is preserved and every record carries its class plus the matched rule or
|
|
11
|
+
* explicit annotation as evidence.
|
|
12
|
+
*/
|
|
13
|
+
export declare function classifyModuleOntologyPolicies(manifest: ModuleManifest, options?: DatabricksOntologyPolicyClassifierOptions): DatabricksOntologyPolicyClassification[];
|
|
14
|
+
/** Input for metric-view DDL generation: one manifest plus its classifications. */
|
|
15
|
+
export interface DatabricksOntologyMetricViewInput {
|
|
16
|
+
manifest: ModuleManifest;
|
|
17
|
+
catalog: string;
|
|
18
|
+
schema: string;
|
|
19
|
+
classifications: readonly DatabricksOntologyPolicyClassification[];
|
|
20
|
+
}
|
|
21
|
+
/** Lowercase snake_case identifier derived from manifest vocabulary. */
|
|
22
|
+
export declare function sqlIdentifier(value: string): string;
|
|
23
|
+
/** One derived metric-view measure (name, expression, comment). */
|
|
24
|
+
export interface DatabricksOntologyMetricViewMeasure {
|
|
25
|
+
name: string;
|
|
26
|
+
expr: string;
|
|
27
|
+
comment: string;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Derive the module's metric-view measures from its threshold/ratio policy
|
|
31
|
+
* classifications. Shared by the DDL generator (P0-2) and the Genie spec
|
|
32
|
+
* derivation (P0-5) so measure naming can never drift between the metric view
|
|
33
|
+
* and the agent's `sqlSnippets.measures`.
|
|
34
|
+
*/
|
|
35
|
+
export declare function deriveModuleMetricViewMeasures(manifest: ModuleManifest, classifications: readonly DatabricksOntologyPolicyClassification[]): DatabricksOntologyMetricViewMeasure[];
|
|
36
|
+
/**
|
|
37
|
+
* Generate the module's UC Metric View DDL statement. Deterministic: identical
|
|
38
|
+
* inputs produce byte-identical DDL, so re-export of an unchanged manifest is
|
|
39
|
+
* idempotent by inspection.
|
|
40
|
+
*/
|
|
41
|
+
export declare function generateModuleMetricViewDdl(input: DatabricksOntologyMetricViewInput): Promise<string>;
|
|
42
|
+
//# sourceMappingURL=ontology-metric-view.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ontology-metric-view.d.ts","sourceRoot":"","sources":["../src/ontology-metric-view.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAwB,MAAM,8BAA8B,CAAC;AAMzF,OAAO,KAAK,EAAE,sCAAsC,EAAE,MAAM,sBAAsB,CAAC;AA2EnF,iDAAiD;AACjD,MAAM,WAAW,yCAAyC;IACxD,uFAAuF;IACvF,mBAAmB,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,SAAS,CAAC;CACrD;AAuED;;;;GAIG;AACH,wBAAgB,8BAA8B,CAC5C,QAAQ,EAAE,cAAc,EACxB,OAAO,GAAE,yCAA8C,GACtD,sCAAsC,EAAE,CAyB1C;AAED,mFAAmF;AACnF,MAAM,WAAW,iCAAiC;IAChD,QAAQ,EAAE,cAAc,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,SAAS,sCAAsC,EAAE,CAAC;CACpE;AAED,wEAAwE;AACxE,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAOnD;AAcD,mEAAmE;AACnE,MAAM,WAAW,mCAAmC;IAClD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAC5C,QAAQ,EAAE,cAAc,EACxB,eAAe,EAAE,SAAS,sCAAsC,EAAE,GACjE,mCAAmC,EAAE,CAuCvC;AAED;;;;GAIG;AACH,wBAAsB,2BAA2B,CAC/C,KAAK,EAAE,iCAAiC,GACvC,OAAO,CAAC,MAAM,CAAC,CAsFjB"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fabric-harness/databricks",
|
|
3
|
-
"version": "7.
|
|
3
|
+
"version": "7.1.1",
|
|
4
4
|
"description": "Databricks tools and filesystem sources for Fabric Harness agents.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Fabric",
|
|
@@ -88,7 +88,7 @@
|
|
|
88
88
|
"@databricks/sdk-uc-tables": "0.21.0",
|
|
89
89
|
"@databricks/sdk-uc-volumes": "0.21.0",
|
|
90
90
|
"@databricks/sdk-vectorsearch": "0.21.0",
|
|
91
|
-
"@fabric-harness/sdk": "^6.
|
|
91
|
+
"@fabric-harness/sdk": "^6.2.0"
|
|
92
92
|
},
|
|
93
93
|
"peerDependencies": {
|
|
94
94
|
"@fabric-harness/node": ">=3.0.0",
|
|
@@ -107,8 +107,8 @@
|
|
|
107
107
|
}
|
|
108
108
|
},
|
|
109
109
|
"devDependencies": {
|
|
110
|
-
"@fabric-harness/node": "^6.
|
|
111
|
-
"@fabricorg/platform": "^0.
|
|
110
|
+
"@fabric-harness/node": "^6.2.0",
|
|
111
|
+
"@fabricorg/platform": "^0.10.0",
|
|
112
112
|
"tsup": "8.5.0",
|
|
113
113
|
"typescript": "^5.8.3",
|
|
114
114
|
"vitest": "^4.1.5"
|