@fabric-harness/databricks 4.2.0 → 4.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-services.js +142 -0
- package/dist/agent-services.js.map +1 -0
- package/dist/agent.d.ts.map +1 -1
- package/dist/agent.js +3754 -3563
- package/dist/agent.js.map +1 -1
- package/dist/ai-gateway.js +51 -0
- package/dist/ai-gateway.js.map +1 -0
- package/dist/ai-search-admin.js +194 -0
- package/dist/ai-search-admin.js.map +1 -0
- package/dist/ai.js +22 -0
- package/dist/ai.js.map +1 -0
- package/dist/app-deployment.d.ts +8 -0
- package/dist/app-deployment.d.ts.map +1 -1
- package/dist/app-user-authorization.d.ts +6 -0
- package/dist/app-user-authorization.d.ts.map +1 -1
- package/dist/app-user-authorization.js +33 -0
- package/dist/app-user-authorization.js.map +1 -1
- package/dist/app.js +143 -0
- package/dist/app.js.map +1 -0
- package/dist/authoring-certification.js +476 -0
- package/dist/authoring-certification.js.map +1 -0
- package/dist/capabilities.d.ts.map +1 -1
- package/dist/capabilities.js +633 -0
- package/dist/capabilities.js.map +1 -0
- package/dist/certification-environment.d.ts.map +1 -1
- package/dist/certification-resource.js +17 -0
- package/dist/certification-resource.js.map +1 -0
- package/dist/certification-sweeper.js +128 -0
- package/dist/certification-sweeper.js.map +1 -0
- package/dist/certification.d.ts +8 -1
- package/dist/certification.d.ts.map +1 -1
- package/dist/certification.js +236 -0
- package/dist/certification.js.map +1 -0
- package/dist/certify.js +93 -7
- package/dist/certify.js.map +1 -1
- package/dist/client.d.ts +58 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.js +184 -0
- package/dist/client.js.map +1 -0
- package/dist/compatibility.js +88 -0
- package/dist/compatibility.js.map +1 -0
- package/dist/consumption.js +85 -0
- package/dist/consumption.js.map +1 -0
- package/dist/cost-budget-actuals.js +316 -0
- package/dist/cost-budget-actuals.js.map +1 -0
- package/dist/embeddings.js +24 -0
- package/dist/embeddings.js.map +1 -0
- package/dist/feature-serving.js +29 -0
- package/dist/feature-serving.js.map +1 -0
- package/dist/genie-agent-mode.js +374 -0
- package/dist/genie-agent-mode.js.map +1 -0
- package/dist/genie-authoring.js +898 -0
- package/dist/genie-authoring.js.map +1 -0
- package/dist/genie.js +405 -0
- package/dist/genie.js.map +1 -0
- package/dist/governance.js +353 -0
- package/dist/governance.js.map +1 -0
- package/dist/identity.js +229 -0
- package/dist/identity.js.map +1 -0
- package/dist/index.d.ts +7 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3913 -3500
- package/dist/index.js.map +1 -1
- package/dist/jobs-authoring.js +570 -0
- package/dist/jobs-authoring.js.map +1 -0
- package/dist/jobs.js +173 -0
- package/dist/jobs.js.map +1 -0
- package/dist/lakebase.js +110 -0
- package/dist/lakebase.js.map +1 -0
- package/dist/lakeflow-authoring.js +115 -0
- package/dist/lakeflow-authoring.js.map +1 -0
- package/dist/lakeflow.js +75 -0
- package/dist/lakeflow.js.map +1 -0
- package/dist/managed-resource.js +197 -0
- package/dist/managed-resource.js.map +1 -0
- package/dist/mlflow-tracing.js +167 -0
- package/dist/mlflow-tracing.js.map +1 -0
- package/dist/mock-provider.js +66 -0
- package/dist/mock-provider.js.map +1 -0
- package/dist/model.js +173 -0
- package/dist/model.js.map +1 -0
- package/dist/persistence.js +82 -0
- package/dist/persistence.js.map +1 -0
- package/dist/rag-chain.js +212 -0
- package/dist/rag-chain.js.map +1 -0
- package/dist/rag-eval.js +151 -0
- package/dist/rag-eval.js.map +1 -0
- package/dist/release-evidence.d.ts +49 -0
- package/dist/release-evidence.d.ts.map +1 -0
- package/dist/secrets-authoring.js +78 -0
- package/dist/secrets-authoring.js.map +1 -0
- package/dist/secrets.js +25 -0
- package/dist/secrets.js.map +1 -0
- package/dist/serving-admin.js +255 -0
- package/dist/serving-admin.js.map +1 -0
- package/dist/serving-usage.js +100 -0
- package/dist/serving-usage.js.map +1 -0
- package/dist/sql-exec.js +36 -0
- package/dist/sql-exec.js.map +1 -0
- package/dist/sql-sandbox.d.ts +18 -0
- package/dist/sql-sandbox.d.ts.map +1 -1
- package/dist/sql-sandbox.js +114 -3
- package/dist/sql-sandbox.js.map +1 -1
- package/dist/telemetry-sink.js +180 -0
- package/dist/telemetry-sink.js.map +1 -0
- package/dist/tools.js +167 -0
- package/dist/tools.js.map +1 -0
- package/dist/unity-catalog-admin.js +166 -0
- package/dist/unity-catalog-admin.js.map +1 -0
- package/dist/vector-search.js +65 -0
- package/dist/vector-search.js.map +1 -0
- package/dist/volumes-attachment-store.js +172 -0
- package/dist/volumes-attachment-store.js.map +1 -0
- package/dist/workspace-authoring.js +78 -0
- package/dist/workspace-authoring.js.map +1 -0
- package/package.json +9 -9
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
/** Create the telemetry tables when absent. Idempotent. */
|
|
2
|
+
export async function ensureDatabricksTelemetryTables(client) {
|
|
3
|
+
await client.query(`CREATE TABLE IF NOT EXISTS fh_submission_telemetry (
|
|
4
|
+
id BIGSERIAL PRIMARY KEY,
|
|
5
|
+
submission_id TEXT NOT NULL,
|
|
6
|
+
event_type TEXT NOT NULL,
|
|
7
|
+
kind TEXT NOT NULL,
|
|
8
|
+
agent TEXT NOT NULL,
|
|
9
|
+
instance_id TEXT NOT NULL,
|
|
10
|
+
session TEXT NOT NULL,
|
|
11
|
+
attempt_id TEXT,
|
|
12
|
+
attempt_count INTEGER,
|
|
13
|
+
outcome TEXT,
|
|
14
|
+
error TEXT,
|
|
15
|
+
duration_ms BIGINT,
|
|
16
|
+
model TEXT,
|
|
17
|
+
input_tokens BIGINT,
|
|
18
|
+
output_tokens BIGINT,
|
|
19
|
+
cost_usd DOUBLE PRECISION,
|
|
20
|
+
tenant_id TEXT,
|
|
21
|
+
principal_kind TEXT,
|
|
22
|
+
principal_id TEXT,
|
|
23
|
+
at BIGINT NOT NULL
|
|
24
|
+
)`);
|
|
25
|
+
await ensureIndex(client, "fh_submission_telemetry", "fh_submission_telemetry_submission_idx", "CREATE INDEX IF NOT EXISTS fh_submission_telemetry_submission_idx ON fh_submission_telemetry (submission_id)");
|
|
26
|
+
await client.query(`CREATE TABLE IF NOT EXISTS fh_lineage (
|
|
27
|
+
id BIGSERIAL PRIMARY KEY,
|
|
28
|
+
submission_id TEXT,
|
|
29
|
+
attempt_id TEXT,
|
|
30
|
+
tool TEXT NOT NULL,
|
|
31
|
+
service TEXT,
|
|
32
|
+
provider TEXT,
|
|
33
|
+
principal TEXT,
|
|
34
|
+
catalog TEXT,
|
|
35
|
+
schema_name TEXT,
|
|
36
|
+
object_name TEXT,
|
|
37
|
+
operation TEXT,
|
|
38
|
+
ok BOOLEAN NOT NULL,
|
|
39
|
+
error TEXT,
|
|
40
|
+
tenant_id TEXT,
|
|
41
|
+
input JSONB,
|
|
42
|
+
started_at TEXT NOT NULL,
|
|
43
|
+
finished_at TEXT NOT NULL
|
|
44
|
+
)`);
|
|
45
|
+
await ensureColumn(client, "fh_lineage", "object_name", "TEXT");
|
|
46
|
+
await ensureColumn(client, "fh_lineage", "operation", "TEXT");
|
|
47
|
+
await ensureIndex(client, "fh_lineage", "fh_lineage_submission_idx", "CREATE INDEX IF NOT EXISTS fh_lineage_submission_idx ON fh_lineage (submission_id)");
|
|
48
|
+
}
|
|
49
|
+
async function ensureColumn(client, table, column, type) {
|
|
50
|
+
const existing = await client.query(`SELECT 1 AS present
|
|
51
|
+
FROM pg_catalog.pg_attribute
|
|
52
|
+
WHERE attrelid = pg_catalog.to_regclass($1)
|
|
53
|
+
AND attname = $2
|
|
54
|
+
AND attnum > 0
|
|
55
|
+
AND NOT attisdropped
|
|
56
|
+
LIMIT 1`, [table, column]);
|
|
57
|
+
if (existing.rows.length > 0)
|
|
58
|
+
return;
|
|
59
|
+
await client.query(`ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS ${column} ${type}`);
|
|
60
|
+
}
|
|
61
|
+
async function ensureIndex(client, table, index, createSql) {
|
|
62
|
+
const existing = await client.query(`SELECT 1 AS present
|
|
63
|
+
FROM pg_catalog.pg_class AS index_relation
|
|
64
|
+
JOIN pg_catalog.pg_index AS index_metadata
|
|
65
|
+
ON index_metadata.indexrelid = index_relation.oid
|
|
66
|
+
WHERE index_metadata.indrelid = pg_catalog.to_regclass($1)
|
|
67
|
+
AND index_relation.relname = $2
|
|
68
|
+
LIMIT 1`, [table, index]);
|
|
69
|
+
if (existing.rows.length > 0)
|
|
70
|
+
return;
|
|
71
|
+
await client.query(createSql);
|
|
72
|
+
}
|
|
73
|
+
export function databricksTelemetry(options) {
|
|
74
|
+
const { client } = options;
|
|
75
|
+
const onError = options.onError ??
|
|
76
|
+
((context, error) => {
|
|
77
|
+
console.error(`[fabric-harness:databricks-telemetry] ${context}:`, error);
|
|
78
|
+
});
|
|
79
|
+
let initialized;
|
|
80
|
+
const ensure = () => {
|
|
81
|
+
if (options.initialize === false)
|
|
82
|
+
return Promise.resolve();
|
|
83
|
+
initialized ??= ensureDatabricksTelemetryTables(client).catch((error) => {
|
|
84
|
+
initialized = undefined;
|
|
85
|
+
throw error;
|
|
86
|
+
});
|
|
87
|
+
return initialized;
|
|
88
|
+
};
|
|
89
|
+
async function writeEvent(event) {
|
|
90
|
+
await ensure();
|
|
91
|
+
const attemptId = "attemptId" in event ? (event.attemptId ?? null) : null;
|
|
92
|
+
const attemptCount = "attemptCount" in event ? (event.attemptCount ?? null) : null;
|
|
93
|
+
const outcome = event.type === "submission_settled" ? event.outcome : null;
|
|
94
|
+
const error = event.type === "submission_settled" ? (event.error ?? null) : null;
|
|
95
|
+
const durationMs = event.type === "submission_settled" ? (event.durationMs ?? null) : null;
|
|
96
|
+
const model = event.type === "submission_usage" ? (event.model ?? null) : null;
|
|
97
|
+
const inputTokens = event.type === "submission_usage" ? (event.inputTokens ?? null) : null;
|
|
98
|
+
const outputTokens = event.type === "submission_usage" ? (event.outputTokens ?? null) : null;
|
|
99
|
+
const costUsd = event.type === "submission_usage" ? (event.costUsd ?? null) : null;
|
|
100
|
+
await client.query(`INSERT INTO fh_submission_telemetry
|
|
101
|
+
(submission_id, event_type, kind, agent, instance_id, session, attempt_id, attempt_count, outcome, error, duration_ms, model, input_tokens, output_tokens, cost_usd, tenant_id, principal_kind, principal_id, at)
|
|
102
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19)`, [
|
|
103
|
+
event.submissionId,
|
|
104
|
+
event.type,
|
|
105
|
+
event.kind,
|
|
106
|
+
event.agent,
|
|
107
|
+
event.instanceId,
|
|
108
|
+
event.session,
|
|
109
|
+
attemptId,
|
|
110
|
+
attemptCount,
|
|
111
|
+
outcome,
|
|
112
|
+
error,
|
|
113
|
+
durationMs,
|
|
114
|
+
model,
|
|
115
|
+
inputTokens,
|
|
116
|
+
outputTokens,
|
|
117
|
+
costUsd,
|
|
118
|
+
event.tenantId ?? null,
|
|
119
|
+
event.actor?.principal?.kind ?? null,
|
|
120
|
+
event.actor?.principal?.id ?? null,
|
|
121
|
+
event.at,
|
|
122
|
+
]);
|
|
123
|
+
}
|
|
124
|
+
async function writeLineage(record) {
|
|
125
|
+
await ensure();
|
|
126
|
+
await client.query(`INSERT INTO fh_lineage
|
|
127
|
+
(submission_id, attempt_id, tool, service, provider, principal, catalog, schema_name, object_name, operation, ok, error, tenant_id, input, started_at, finished_at)
|
|
128
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14::jsonb, $15, $16)`, [
|
|
129
|
+
record.submissionId ?? null,
|
|
130
|
+
record.attemptId ?? null,
|
|
131
|
+
record.tool,
|
|
132
|
+
record.service ?? null,
|
|
133
|
+
record.provider ?? null,
|
|
134
|
+
record.principal ?? null,
|
|
135
|
+
record.catalog ?? null,
|
|
136
|
+
record.schema ?? null,
|
|
137
|
+
record.object ?? null,
|
|
138
|
+
record.operation ?? null,
|
|
139
|
+
record.ok,
|
|
140
|
+
record.error ?? null,
|
|
141
|
+
record.tenantId ?? null,
|
|
142
|
+
JSON.stringify(record.input ?? {}),
|
|
143
|
+
record.startedAt,
|
|
144
|
+
record.finishedAt,
|
|
145
|
+
]);
|
|
146
|
+
}
|
|
147
|
+
return {
|
|
148
|
+
sink: {
|
|
149
|
+
record(event) {
|
|
150
|
+
return writeEvent(event);
|
|
151
|
+
},
|
|
152
|
+
},
|
|
153
|
+
onLineage(record) {
|
|
154
|
+
void writeLineage(record).catch((error) => onError(`lineage write failed for ${record.tool}`, error));
|
|
155
|
+
},
|
|
156
|
+
migrate: ensure,
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
/** Join governed object access to the submission actor, outcome, and model/tool cost. */
|
|
160
|
+
export async function queryDatabricksLineageEvidence(client, submissionId) {
|
|
161
|
+
const result = await client.query(`SELECT l.submission_id, COALESCE(l.principal, MAX(t.principal_id)) AS actor,
|
|
162
|
+
l.object_name, l.operation, l.ok, COALESCE(SUM(t.cost_usd), 0) AS estimated_cost_usd,
|
|
163
|
+
l.started_at, l.finished_at
|
|
164
|
+
FROM fh_lineage l
|
|
165
|
+
LEFT JOIN fh_submission_telemetry t ON t.submission_id = l.submission_id
|
|
166
|
+
WHERE l.submission_id = $1
|
|
167
|
+
GROUP BY l.id, l.submission_id, l.principal, l.object_name, l.operation, l.ok, l.started_at, l.finished_at
|
|
168
|
+
ORDER BY l.id`, [submissionId]);
|
|
169
|
+
return result.rows.map((row) => ({
|
|
170
|
+
submissionId: String(row.submission_id),
|
|
171
|
+
...(row.actor ? { actor: String(row.actor) } : {}),
|
|
172
|
+
...(row.object_name ? { object: String(row.object_name) } : {}),
|
|
173
|
+
...(row.operation ? { operation: String(row.operation) } : {}),
|
|
174
|
+
outcome: row.ok ? "succeeded" : "failed",
|
|
175
|
+
estimatedCostUsd: Number(row.estimated_cost_usd ?? 0),
|
|
176
|
+
startedAt: String(row.started_at),
|
|
177
|
+
finishedAt: String(row.finished_at),
|
|
178
|
+
}));
|
|
179
|
+
}
|
|
180
|
+
//# sourceMappingURL=telemetry-sink.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"telemetry-sink.js","sourceRoot":"","sources":["../src/telemetry-sink.ts"],"names":[],"mappings":"AA4BA,2DAA2D;AAC3D,MAAM,CAAC,KAAK,UAAU,+BAA+B,CAAC,MAAsB;IAC1E,MAAM,MAAM,CAAC,KAAK,CAAC;;;;;;;;;;;;;;;;;;;;;IAqBjB,CAAC,CAAC;IACJ,MAAM,WAAW,CACf,MAAM,EACN,yBAAyB,EACzB,wCAAwC,EACxC,8GAA8G,CAC/G,CAAC;IACF,MAAM,MAAM,CAAC,KAAK,CAAC;;;;;;;;;;;;;;;;;;IAkBjB,CAAC,CAAC;IACJ,MAAM,YAAY,CAAC,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC;IAChE,MAAM,YAAY,CAAC,MAAM,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;IAC9D,MAAM,WAAW,CACf,MAAM,EACN,YAAY,EACZ,2BAA2B,EAC3B,oFAAoF,CACrF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,MAAsB,EACtB,KAAa,EACb,MAAc,EACd,IAAY;IAEZ,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,KAAK,CACjC;;;;;;cAMU,EACV,CAAC,KAAK,EAAE,MAAM,CAAC,CAChB,CAAC;IACF,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO;IACrC,MAAM,MAAM,CAAC,KAAK,CAAC,eAAe,KAAK,6BAA6B,MAAM,IAAI,IAAI,EAAE,CAAC,CAAC;AACxF,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,MAAsB,EACtB,KAAa,EACb,KAAa,EACb,SAAiB;IAEjB,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,KAAK,CACjC;;;;;;cAMU,EACV,CAAC,KAAK,EAAE,KAAK,CAAC,CACf,CAAC;IACF,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO;IACrC,MAAM,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,OAAmC;IACrE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAC3B,MAAM,OAAO,GACX,OAAO,CAAC,OAAO;QACf,CAAC,CAAC,OAAe,EAAE,KAAc,EAAE,EAAE;YACnC,OAAO,CAAC,KAAK,CAAC,yCAAyC,OAAO,GAAG,EAAE,KAAK,CAAC,CAAC;QAC5E,CAAC,CAAC,CAAC;IAEL,IAAI,WAAsC,CAAC;IAC3C,MAAM,MAAM,GAAG,GAAkB,EAAE;QACjC,IAAI,OAAO,CAAC,UAAU,KAAK,KAAK;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3D,WAAW,KAAK,+BAA+B,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACtE,WAAW,GAAG,SAAS,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC,CAAC,CAAC;QACH,OAAO,WAAW,CAAC;IACrB,CAAC,CAAC;IAEF,KAAK,UAAU,UAAU,CAAC,KAA+B;QACvD,MAAM,MAAM,EAAE,CAAC;QACf,MAAM,SAAS,GAAG,WAAW,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC1E,MAAM,YAAY,GAAG,cAAc,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACnF,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,KAAK,oBAAoB,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;QAC3E,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,KAAK,oBAAoB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACjF,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,KAAK,oBAAoB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC3F,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,KAAK,kBAAkB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC/E,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,KAAK,kBAAkB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC3F,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,KAAK,kBAAkB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC7F,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,KAAK,kBAAkB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACnF,MAAM,MAAM,CAAC,KAAK,CAChB;;qGAE+F,EAC/F;YACE,KAAK,CAAC,YAAY;YAClB,KAAK,CAAC,IAAI;YACV,KAAK,CAAC,IAAI;YACV,KAAK,CAAC,KAAK;YACX,KAAK,CAAC,UAAU;YAChB,KAAK,CAAC,OAAO;YACb,SAAS;YACT,YAAY;YACZ,OAAO;YACP,KAAK;YACL,UAAU;YACV,KAAK;YACL,WAAW;YACX,YAAY;YACZ,OAAO;YACP,KAAK,CAAC,QAAQ,IAAI,IAAI;YACtB,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,IAAI,IAAI;YACpC,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,IAAI,IAAI;YAClC,KAAK,CAAC,EAAE;SACT,CACF,CAAC;IACJ,CAAC;IAED,KAAK,UAAU,YAAY,CAAC,MAA+B;QACzD,MAAM,MAAM,EAAE,CAAC;QACf,MAAM,MAAM,CAAC,KAAK,CAChB;;6FAEuF,EACvF;YACE,MAAM,CAAC,YAAY,IAAI,IAAI;YAC3B,MAAM,CAAC,SAAS,IAAI,IAAI;YACxB,MAAM,CAAC,IAAI;YACX,MAAM,CAAC,OAAO,IAAI,IAAI;YACtB,MAAM,CAAC,QAAQ,IAAI,IAAI;YACvB,MAAM,CAAC,SAAS,IAAI,IAAI;YACxB,MAAM,CAAC,OAAO,IAAI,IAAI;YACtB,MAAM,CAAC,MAAM,IAAI,IAAI;YACrB,MAAM,CAAC,MAAM,IAAI,IAAI;YACrB,MAAM,CAAC,SAAS,IAAI,IAAI;YACxB,MAAM,CAAC,EAAE;YACT,MAAM,CAAC,KAAK,IAAI,IAAI;YACpB,MAAM,CAAC,QAAQ,IAAI,IAAI;YACvB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;YAClC,MAAM,CAAC,SAAS;YAChB,MAAM,CAAC,UAAU;SAClB,CACF,CAAC;IACJ,CAAC;IAED,OAAO;QACL,IAAI,EAAE;YACJ,MAAM,CAAC,KAAK;gBACV,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC;YAC3B,CAAC;SACF;QACD,SAAS,CAAC,MAAM;YACd,KAAK,YAAY,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CACxC,OAAO,CAAC,4BAA4B,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC,CAC1D,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,MAAM;KAChB,CAAC;AACJ,CAAC;AAaD,yFAAyF;AACzF,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAClD,MAAsB,EACtB,YAAoB;IAEpB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAC/B;;;;;;;oBAOgB,EAChB,CAAC,YAAY,CAAC,CACf,CAAC;IACF,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAC/B,YAAY,EAAE,MAAM,CAAC,GAAG,CAAC,aAAa,CAAC;QACvC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClD,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/D,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ;QACxC,gBAAgB,EAAE,MAAM,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC,CAAC;QACrD,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC;QACjC,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC;KACpC,CAAC,CAAC,CAAC;AACN,CAAC"}
|
package/dist/tools.js
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import { databricksJobs } from './jobs.js';
|
|
2
|
+
export function databricksSqlTool(client, options) {
|
|
3
|
+
return {
|
|
4
|
+
name: options.name ?? 'databricks_sql',
|
|
5
|
+
description: options.description ?? 'Run a SQL statement on a Databricks SQL Warehouse.',
|
|
6
|
+
inputSchema: {
|
|
7
|
+
type: 'object',
|
|
8
|
+
additionalProperties: false,
|
|
9
|
+
properties: {
|
|
10
|
+
statement: { type: 'string' },
|
|
11
|
+
catalog: { type: 'string' },
|
|
12
|
+
schema: { type: 'string' },
|
|
13
|
+
},
|
|
14
|
+
required: ['statement'],
|
|
15
|
+
},
|
|
16
|
+
metadata: { effect: 'execute', provider: 'databricks', service: 'sql' },
|
|
17
|
+
execute: (input) => client.post('/api/2.0/sql/statements', {
|
|
18
|
+
warehouse_id: options.warehouseId,
|
|
19
|
+
statement: input.statement,
|
|
20
|
+
wait_timeout: options.waitTimeout ?? '30s',
|
|
21
|
+
disposition: options.disposition ?? 'INLINE',
|
|
22
|
+
...(input.catalog ? { catalog: input.catalog } : {}),
|
|
23
|
+
...(input.schema ? { schema: input.schema } : {}),
|
|
24
|
+
}),
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
export function databricksRunJobTool(client, options = {}) {
|
|
28
|
+
return {
|
|
29
|
+
name: options.name ?? 'databricks_run_job',
|
|
30
|
+
description: options.description ?? 'Trigger an existing Databricks job by job id.',
|
|
31
|
+
inputSchema: {
|
|
32
|
+
type: 'object',
|
|
33
|
+
additionalProperties: false,
|
|
34
|
+
properties: {
|
|
35
|
+
jobId: { type: 'number' },
|
|
36
|
+
idempotencyToken: { type: 'string' },
|
|
37
|
+
notebookParams: { type: 'object', additionalProperties: true },
|
|
38
|
+
pythonParams: { type: 'array', items: { type: 'string' } },
|
|
39
|
+
jarParams: { type: 'array', items: { type: 'string' } },
|
|
40
|
+
},
|
|
41
|
+
required: ['jobId'],
|
|
42
|
+
},
|
|
43
|
+
metadata: { effect: 'execute', provider: 'databricks', service: 'jobs' },
|
|
44
|
+
execute: (input) => databricksJobs(client).runJob(input),
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
export function databricksNotebookTool(client, options = {}) {
|
|
48
|
+
return {
|
|
49
|
+
name: options.name ?? 'databricks_run_notebook',
|
|
50
|
+
description: options.description ?? 'Submit a one-off Databricks notebook run.',
|
|
51
|
+
inputSchema: {
|
|
52
|
+
type: 'object',
|
|
53
|
+
additionalProperties: false,
|
|
54
|
+
properties: {
|
|
55
|
+
notebookPath: { type: 'string' },
|
|
56
|
+
idempotencyToken: { type: 'string' },
|
|
57
|
+
baseParameters: { type: 'object', additionalProperties: true },
|
|
58
|
+
runName: { type: 'string' },
|
|
59
|
+
},
|
|
60
|
+
required: ['notebookPath'],
|
|
61
|
+
},
|
|
62
|
+
metadata: { effect: 'execute', provider: 'databricks', service: 'jobs' },
|
|
63
|
+
execute: (input) => databricksJobs(client).submitNotebook({ ...input, ...(options.existingClusterId ? { existingClusterId: options.existingClusterId } : {}) }),
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
export function databricksMlflowLogMetricTool(client, options = {}) {
|
|
67
|
+
return {
|
|
68
|
+
name: options.name ?? 'databricks_mlflow_log_metric',
|
|
69
|
+
description: options.description ?? 'Log a metric to an MLflow run hosted in Databricks.',
|
|
70
|
+
inputSchema: {
|
|
71
|
+
type: 'object',
|
|
72
|
+
additionalProperties: false,
|
|
73
|
+
properties: {
|
|
74
|
+
runId: { type: 'string' },
|
|
75
|
+
key: { type: 'string' },
|
|
76
|
+
value: { type: 'number' },
|
|
77
|
+
step: { type: 'number' },
|
|
78
|
+
timestamp: { type: 'number' },
|
|
79
|
+
},
|
|
80
|
+
required: ['runId', 'key', 'value'],
|
|
81
|
+
},
|
|
82
|
+
metadata: { effect: 'write', provider: 'databricks', service: 'mlflow' },
|
|
83
|
+
execute: (input) => client.post('/api/2.0/mlflow/runs/log-metric', {
|
|
84
|
+
run_id: input.runId,
|
|
85
|
+
key: input.key,
|
|
86
|
+
value: input.value,
|
|
87
|
+
timestamp: input.timestamp ?? Date.now(),
|
|
88
|
+
...(input.step !== undefined ? { step: input.step } : {}),
|
|
89
|
+
}, {}, { retry: 'safe' }),
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
export function databricksMlflowLogParamTool(client, options = {}) {
|
|
93
|
+
return {
|
|
94
|
+
name: options.name ?? 'databricks_mlflow_log_param',
|
|
95
|
+
description: options.description ?? 'Log a parameter to an MLflow run hosted in Databricks.',
|
|
96
|
+
inputSchema: {
|
|
97
|
+
type: 'object',
|
|
98
|
+
additionalProperties: false,
|
|
99
|
+
properties: { runId: { type: 'string' }, key: { type: 'string' }, value: { type: 'string' } },
|
|
100
|
+
required: ['runId', 'key', 'value'],
|
|
101
|
+
},
|
|
102
|
+
metadata: { effect: 'write', provider: 'databricks', service: 'mlflow' },
|
|
103
|
+
execute: (input) => client.post('/api/2.0/mlflow/runs/log-parameter', { run_id: input.runId, key: input.key, value: input.value }, {}, { retry: 'safe' }),
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
export function unityCatalogTablesTool(client, options = {}) {
|
|
107
|
+
return {
|
|
108
|
+
name: options.name ?? 'databricks_unity_catalog_tables',
|
|
109
|
+
description: options.description ?? 'List Unity Catalog tables in a catalog and schema.',
|
|
110
|
+
inputSchema: {
|
|
111
|
+
type: 'object',
|
|
112
|
+
additionalProperties: false,
|
|
113
|
+
properties: { catalog: { type: 'string' }, schema: { type: 'string' } },
|
|
114
|
+
required: ['catalog', 'schema'],
|
|
115
|
+
},
|
|
116
|
+
metadata: { effect: 'read', provider: 'databricks', service: 'unity-catalog' },
|
|
117
|
+
execute: (input) => client.get('/api/2.1/unity-catalog/tables', { catalog_name: input.catalog, schema_name: input.schema }),
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
export function databricksTableInfoTool(client, options = {}) {
|
|
121
|
+
return {
|
|
122
|
+
name: options.name ?? 'databricks_table_info',
|
|
123
|
+
description: options.description ?? 'Describe a Unity Catalog table (columns, types, comment) by its full name (catalog.schema.table).',
|
|
124
|
+
inputSchema: {
|
|
125
|
+
type: 'object',
|
|
126
|
+
additionalProperties: false,
|
|
127
|
+
properties: { fullName: { type: 'string' } },
|
|
128
|
+
required: ['fullName'],
|
|
129
|
+
},
|
|
130
|
+
metadata: { effect: 'read', provider: 'databricks', service: 'unity-catalog' },
|
|
131
|
+
execute: (input) => client.get(`/api/2.1/unity-catalog/tables/${encodeURIComponent(input.fullName)}`),
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
export function databricksWorkspaceSource(client, rootPath, options = {}) {
|
|
135
|
+
const include = options.include ?? (() => true);
|
|
136
|
+
return {
|
|
137
|
+
name: options.name ?? `databricks-workspace:${rootPath}`,
|
|
138
|
+
async *entries() {
|
|
139
|
+
yield* walkWorkspace(client, rootPath, rootPath, include, options.recursive ?? true, options.format ?? 'SOURCE');
|
|
140
|
+
},
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
async function* walkWorkspace(client, root, current, include, recursive, format) {
|
|
144
|
+
const listing = await client.get('/api/2.0/workspace/list', { path: current });
|
|
145
|
+
for (const object of listing.objects ?? []) {
|
|
146
|
+
if (object.object_type === 'DIRECTORY') {
|
|
147
|
+
if (recursive)
|
|
148
|
+
yield* walkWorkspace(client, root, object.path, include, recursive, format);
|
|
149
|
+
continue;
|
|
150
|
+
}
|
|
151
|
+
const relative = object.path.slice(root.length).replace(/^\/+/, '');
|
|
152
|
+
if (!relative || !include(relative))
|
|
153
|
+
continue;
|
|
154
|
+
const exported = await client.get('/api/2.0/workspace/export', { path: object.path, format });
|
|
155
|
+
yield { path: relative, content: exported.content ? base64ToBytes(exported.content) : new Uint8Array() };
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
function base64ToBytes(value) {
|
|
159
|
+
if (typeof Buffer !== 'undefined')
|
|
160
|
+
return new Uint8Array(Buffer.from(value, 'base64'));
|
|
161
|
+
const binary = atob(value);
|
|
162
|
+
const bytes = new Uint8Array(binary.length);
|
|
163
|
+
for (let i = 0; i < binary.length; i += 1)
|
|
164
|
+
bytes[i] = binary.charCodeAt(i);
|
|
165
|
+
return bytes;
|
|
166
|
+
}
|
|
167
|
+
//# sourceMappingURL=tools.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tools.js","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAA6B,MAAM,WAAW,CAAC;AAUtE,MAAM,UAAU,iBAAiB,CAAC,MAA4B,EAAE,OAAiC;IAC/F,OAAO;QACL,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,gBAAgB;QACtC,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,oDAAoD;QACxF,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,oBAAoB,EAAE,KAAK;YAC3B,UAAU,EAAE;gBACV,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC7B,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC3B,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aAC3B;YACD,QAAQ,EAAE,CAAC,WAAW,CAAC;SACxB;QACD,QAAQ,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,KAAK,EAAE;QACvE,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE;YACzD,YAAY,EAAE,OAAO,CAAC,WAAW;YACjC,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,YAAY,EAAE,OAAO,CAAC,WAAW,IAAI,KAAK;YAC1C,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,QAAQ;YAC5C,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpD,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClD,CAAC;KACH,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,MAA4B,EAAE,UAAmD,EAAE;IACtH,OAAO;QACL,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,oBAAoB;QAC1C,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,+CAA+C;QACnF,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,oBAAoB,EAAE,KAAK;YAC3B,UAAU,EAAE;gBACV,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACzB,gBAAgB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACpC,cAAc,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,oBAAoB,EAAE,IAAI,EAAE;gBAC9D,YAAY,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;gBAC1D,SAAS,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;aACxD;YACD,QAAQ,EAAE,CAAC,OAAO,CAAC;SACpB;QACD,QAAQ,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,EAAE;QACxE,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;KACzD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,MAA4B,EAAE,UAA+E,EAAE;IACpJ,OAAO;QACL,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,yBAAyB;QAC/C,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,2CAA2C;QAC/E,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,oBAAoB,EAAE,KAAK;YAC3B,UAAU,EAAE;gBACV,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAChC,gBAAgB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACpC,cAAc,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,oBAAoB,EAAE,IAAI,EAAE;gBAC9D,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aAC5B;YACD,QAAQ,EAAE,CAAC,cAAc,CAAC;SAC3B;QACD,QAAQ,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,EAAE;QACxE,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,EAAE,GAAG,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;KAChK,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,MAA4B,EAAE,UAAmD,EAAE;IAC/H,OAAO;QACL,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,8BAA8B;QACpD,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,qDAAqD;QACzF,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,oBAAoB,EAAE,KAAK;YAC3B,UAAU,EAAE;gBACV,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACzB,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACvB,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACzB,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACxB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aAC9B;YACD,QAAQ,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC;SACpC;QACD,QAAQ,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,QAAQ,EAAE;QACxE,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,iCAAiC,EAAE;YACjE,MAAM,EAAE,KAAK,CAAC,KAAK;YACnB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE;YACxC,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC1D,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;KAC1B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,4BAA4B,CAAC,MAA4B,EAAE,UAAmD,EAAE;IAC9H,OAAO;QACL,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,6BAA6B;QACnD,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,wDAAwD;QAC5F,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,oBAAoB,EAAE,KAAK;YAC3B,UAAU,EAAE,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;YAC7F,QAAQ,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC;SACpC;QACD,QAAQ,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,QAAQ,EAAE;QACxE,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAC7B,oCAAoC,EACpC,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,EAC3D,EAAE,EACF,EAAE,KAAK,EAAE,MAAM,EAAE,CAClB;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,MAA4B,EAAE,UAAmD,EAAE;IACxH,OAAO;QACL,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,iCAAiC;QACvD,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,oDAAoD;QACxF,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,oBAAoB,EAAE,KAAK;YAC3B,UAAU,EAAE,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;YACvE,QAAQ,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC;SAChC;QACD,QAAQ,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe,EAAE;QAC9E,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,+BAA+B,EAAE,EAAE,YAAY,EAAE,KAAK,CAAC,OAAO,EAAE,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;KAC5H,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,MAA4B,EAAE,UAAmD,EAAE;IACzH,OAAO;QACL,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,uBAAuB;QAC7C,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,mGAAmG;QACvI,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,oBAAoB,EAAE,KAAK;YAC3B,UAAU,EAAE,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;YAC5C,QAAQ,EAAE,CAAC,UAAU,CAAC;SACvB;QACD,QAAQ,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe,EAAE;QAC9E,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,iCAAiC,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;KACtG,CAAC;AACJ,CAAC;AASD,MAAM,UAAU,yBAAyB,CAAC,MAA4B,EAAE,QAAgB,EAAE,UAA4C,EAAE;IACtI,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAChD,OAAO;QACL,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,wBAAwB,QAAQ,EAAE;QACxD,KAAK,CAAC,CAAC,OAAO;YACZ,KAAK,CAAC,CAAC,aAAa,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI,EAAE,OAAO,CAAC,MAAM,IAAI,QAAQ,CAAC,CAAC;QACnH,CAAC;KACF,CAAC;AACJ,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,aAAa,CAAC,MAA4B,EAAE,IAAY,EAAE,OAAe,EAAE,OAAkC,EAAE,SAAkB,EAAE,MAAc;IAC/J,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,GAAG,CAA8D,yBAAyB,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;IAC5I,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QAC3C,IAAI,MAAM,CAAC,WAAW,KAAK,WAAW,EAAE,CAAC;YACvC,IAAI,SAAS;gBAAE,KAAK,CAAC,CAAC,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YAC3F,SAAS;QACX,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACpE,IAAI,CAAC,QAAQ,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;YAAE,SAAS;QAC9C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,CAAuB,2BAA2B,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QACpH,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,UAAU,EAAE,EAAE,CAAC;IAC3G,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IAClC,IAAI,OAAO,MAAM,KAAK,WAAW;QAAE,OAAO,IAAI,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;IACvF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3B,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC;QAAE,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAC3E,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
const MANAGED_PROPERTY = "fabric-harness.managed";
|
|
2
|
+
export class DatabricksUnityCatalogAdmin {
|
|
3
|
+
client;
|
|
4
|
+
constructor(client) {
|
|
5
|
+
this.client = client;
|
|
6
|
+
}
|
|
7
|
+
getPermissions(securableType, fullName) {
|
|
8
|
+
return this.client.get(`/api/2.1/unity-catalog/permissions/${encodeURIComponent(securableType)}/${encodeURIComponent(fullName)}`);
|
|
9
|
+
}
|
|
10
|
+
updatePermissions(securableType, fullName, changes) {
|
|
11
|
+
return this.client.patch(`/api/2.1/unity-catalog/permissions/${encodeURIComponent(securableType)}/${encodeURIComponent(fullName)}`, { changes }, {}, { retry: "never" });
|
|
12
|
+
}
|
|
13
|
+
createCatalog(input) {
|
|
14
|
+
return this.client.post("/api/2.1/unity-catalog/catalogs", {
|
|
15
|
+
name: input.name,
|
|
16
|
+
...(input.comment ? { comment: input.comment } : {}),
|
|
17
|
+
...(input.storageRoot ? { storage_root: input.storageRoot } : {}),
|
|
18
|
+
properties: { ...(input.properties ?? {}), [MANAGED_PROPERTY]: "true" },
|
|
19
|
+
}, {}, { retry: "never" });
|
|
20
|
+
}
|
|
21
|
+
createSchema(input) {
|
|
22
|
+
return this.client.post("/api/2.1/unity-catalog/schemas", {
|
|
23
|
+
name: input.name,
|
|
24
|
+
catalog_name: input.catalogName,
|
|
25
|
+
...(input.comment ? { comment: input.comment } : {}),
|
|
26
|
+
properties: { ...(input.properties ?? {}), [MANAGED_PROPERTY]: "true" },
|
|
27
|
+
}, {}, { retry: "never" });
|
|
28
|
+
}
|
|
29
|
+
createVolume(input) {
|
|
30
|
+
return this.client.post("/api/2.1/unity-catalog/volumes", {
|
|
31
|
+
catalog_name: input.catalogName,
|
|
32
|
+
schema_name: input.schemaName,
|
|
33
|
+
name: input.name,
|
|
34
|
+
volume_type: input.volumeType ?? "MANAGED",
|
|
35
|
+
...(input.storageLocation ? { storage_location: input.storageLocation } : {}),
|
|
36
|
+
comment: `[fabric-harness:managed]${input.comment ? ` ${input.comment}` : ""}`,
|
|
37
|
+
}, {}, { retry: "never" });
|
|
38
|
+
}
|
|
39
|
+
deleteCatalog(name, force = false) {
|
|
40
|
+
return this.client.delete(`/api/2.1/unity-catalog/catalogs/${encodeURIComponent(name)}`, undefined, { force }, { retry: "never" });
|
|
41
|
+
}
|
|
42
|
+
deleteSchema(fullName, force = false) {
|
|
43
|
+
return this.client.delete(`/api/2.1/unity-catalog/schemas/${encodeURIComponent(fullName)}`, undefined, { force }, { retry: "never" });
|
|
44
|
+
}
|
|
45
|
+
deleteVolume(fullName) {
|
|
46
|
+
return this.client.delete(`/api/2.1/unity-catalog/volumes/${encodeURIComponent(fullName)}`, undefined, {}, { retry: "never" });
|
|
47
|
+
}
|
|
48
|
+
async deleteManaged(kind, fullName) {
|
|
49
|
+
const row = await this.client.get(`/api/2.1/unity-catalog/${kind === "catalog" ? "catalogs" : kind === "schema" ? "schemas" : "volumes"}/${encodeURIComponent(fullName)}`);
|
|
50
|
+
const properties = row.properties;
|
|
51
|
+
const managed = kind === "volume"
|
|
52
|
+
? typeof row.comment === "string" && row.comment.startsWith("[fabric-harness:managed]")
|
|
53
|
+
: properties?.[MANAGED_PROPERTY] === "true";
|
|
54
|
+
if (!managed)
|
|
55
|
+
throw new Error(`databricks UC: refusing to delete unmanaged ${kind} "${fullName}".`);
|
|
56
|
+
if (kind === "catalog")
|
|
57
|
+
return this.deleteCatalog(fullName, false);
|
|
58
|
+
if (kind === "schema")
|
|
59
|
+
return this.deleteSchema(fullName, false);
|
|
60
|
+
return this.deleteVolume(fullName);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
export function databricksUnityCatalogAdminTools(client, options = {}) {
|
|
64
|
+
const api = new DatabricksUnityCatalogAdmin(client);
|
|
65
|
+
const tools = [
|
|
66
|
+
tool("databricks_grant", "write", permissionSchema(), [{ kind: "securable", path: "/fullName", catalogQualified: true }], (i) => api.updatePermissions(i.securableType, i.fullName, [
|
|
67
|
+
{ principal: i.principal, add: i.privileges },
|
|
68
|
+
])),
|
|
69
|
+
tool("databricks_revoke", "write", permissionSchema(), [{ kind: "securable", path: "/fullName", catalogQualified: true }], (i) => api.updatePermissions(i.securableType, i.fullName, [
|
|
70
|
+
{ principal: i.principal, remove: i.privileges },
|
|
71
|
+
])),
|
|
72
|
+
tool("databricks_create_catalog", "write", catalogSchema(), [{ kind: "catalog", path: "/name", catalogQualified: true }], (i) => api.createCatalog(i)),
|
|
73
|
+
tool("databricks_create_schema", "write", schemaSchema(), [
|
|
74
|
+
{ kind: "catalog", path: "/catalogName", catalogQualified: true },
|
|
75
|
+
{ kind: "schema", path: "/name" },
|
|
76
|
+
], (i) => api.createSchema(i)),
|
|
77
|
+
tool("databricks_create_volume", "write", volumeSchema(), [
|
|
78
|
+
{ kind: "catalog", path: "/catalogName", catalogQualified: true },
|
|
79
|
+
{ kind: "schema", path: "/schemaName" },
|
|
80
|
+
{ kind: "volume", path: "/name" },
|
|
81
|
+
], (i) => api.createVolume(i)),
|
|
82
|
+
];
|
|
83
|
+
if (options.destructive)
|
|
84
|
+
tools.push(tool("databricks_delete_uc_object", "write", deleteSchema(), [{ kind: "securable", path: "/fullName", catalogQualified: true }], (i) => api.deleteManaged(i.kind, i.fullName)));
|
|
85
|
+
return tools;
|
|
86
|
+
}
|
|
87
|
+
function tool(name, effect, inputSchema, resources, execute) {
|
|
88
|
+
return {
|
|
89
|
+
name,
|
|
90
|
+
description: name.replaceAll("_", " "),
|
|
91
|
+
inputSchema,
|
|
92
|
+
metadata: {
|
|
93
|
+
effect,
|
|
94
|
+
provider: "databricks",
|
|
95
|
+
service: "unity-catalog",
|
|
96
|
+
governance: { resources },
|
|
97
|
+
},
|
|
98
|
+
execute,
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
function permissionSchema() {
|
|
102
|
+
return {
|
|
103
|
+
type: "object",
|
|
104
|
+
additionalProperties: false,
|
|
105
|
+
properties: {
|
|
106
|
+
securableType: { type: "string" },
|
|
107
|
+
fullName: { type: "string" },
|
|
108
|
+
principal: { type: "string" },
|
|
109
|
+
privileges: { type: "array", items: { type: "string" } },
|
|
110
|
+
},
|
|
111
|
+
required: ["securableType", "fullName", "principal", "privileges"],
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
function catalogSchema() {
|
|
115
|
+
return {
|
|
116
|
+
type: "object",
|
|
117
|
+
additionalProperties: false,
|
|
118
|
+
properties: {
|
|
119
|
+
name: { type: "string" },
|
|
120
|
+
comment: { type: "string" },
|
|
121
|
+
storageRoot: { type: "string" },
|
|
122
|
+
properties: { type: "object", additionalProperties: { type: "string" } },
|
|
123
|
+
},
|
|
124
|
+
required: ["name"],
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
function schemaSchema() {
|
|
128
|
+
return {
|
|
129
|
+
type: "object",
|
|
130
|
+
additionalProperties: false,
|
|
131
|
+
properties: {
|
|
132
|
+
catalogName: { type: "string" },
|
|
133
|
+
name: { type: "string" },
|
|
134
|
+
comment: { type: "string" },
|
|
135
|
+
properties: { type: "object", additionalProperties: { type: "string" } },
|
|
136
|
+
},
|
|
137
|
+
required: ["catalogName", "name"],
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
function volumeSchema() {
|
|
141
|
+
return {
|
|
142
|
+
type: "object",
|
|
143
|
+
additionalProperties: false,
|
|
144
|
+
properties: {
|
|
145
|
+
catalogName: { type: "string" },
|
|
146
|
+
schemaName: { type: "string" },
|
|
147
|
+
name: { type: "string" },
|
|
148
|
+
volumeType: { type: "string", enum: ["MANAGED", "EXTERNAL"] },
|
|
149
|
+
storageLocation: { type: "string" },
|
|
150
|
+
comment: { type: "string" },
|
|
151
|
+
},
|
|
152
|
+
required: ["catalogName", "schemaName", "name"],
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
function deleteSchema() {
|
|
156
|
+
return {
|
|
157
|
+
type: "object",
|
|
158
|
+
additionalProperties: false,
|
|
159
|
+
properties: {
|
|
160
|
+
kind: { type: "string", enum: ["catalog", "schema", "volume"] },
|
|
161
|
+
fullName: { type: "string" },
|
|
162
|
+
},
|
|
163
|
+
required: ["kind", "fullName"],
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
//# sourceMappingURL=unity-catalog-admin.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"unity-catalog-admin.js","sourceRoot":"","sources":["../src/unity-catalog-admin.ts"],"names":[],"mappings":"AAeA,MAAM,gBAAgB,GAAG,wBAAwB,CAAC;AAElD,MAAM,OAAO,2BAA2B;IACT;IAA7B,YAA6B,MAA4B;QAA5B,WAAM,GAAN,MAAM,CAAsB;IAAG,CAAC;IAC7D,cAAc,CAAC,aAAsC,EAAE,QAAgB;QACrE,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CACpB,sCAAsC,kBAAkB,CAAC,aAAa,CAAC,IAAI,kBAAkB,CAAC,QAAQ,CAAC,EAAE,CAC1G,CAAC;IACJ,CAAC;IACD,iBAAiB,CACf,aAAsC,EACtC,QAAgB,EAChB,OAAqC;QAErC,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CACtB,sCAAsC,kBAAkB,CAAC,aAAa,CAAC,IAAI,kBAAkB,CAAC,QAAQ,CAAC,EAAE,EACzG,EAAE,OAAO,EAAE,EACX,EAAE,EACF,EAAE,KAAK,EAAE,OAAO,EAAE,CACnB,CAAC;IACJ,CAAC;IACD,aAAa,CAAC,KAKb;QACC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CACrB,iCAAiC,EACjC;YACE,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpD,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,UAAU,EAAE,EAAE,GAAG,CAAC,KAAK,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,CAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE;SACxE,EACD,EAAE,EACF,EAAE,KAAK,EAAE,OAAO,EAAE,CACnB,CAAC;IACJ,CAAC;IACD,YAAY,CAAC,KAKZ;QACC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CACrB,gCAAgC,EAChC;YACE,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,YAAY,EAAE,KAAK,CAAC,WAAW;YAC/B,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpD,UAAU,EAAE,EAAE,GAAG,CAAC,KAAK,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,CAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE;SACxE,EACD,EAAE,EACF,EAAE,KAAK,EAAE,OAAO,EAAE,CACnB,CAAC;IACJ,CAAC;IACD,YAAY,CAAC,KAOZ;QACC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CACrB,gCAAgC,EAChC;YACE,YAAY,EAAE,KAAK,CAAC,WAAW;YAC/B,WAAW,EAAE,KAAK,CAAC,UAAU;YAC7B,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,WAAW,EAAE,KAAK,CAAC,UAAU,IAAI,SAAS;YAC1C,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7E,OAAO,EAAE,2BAA2B,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE;SAC/E,EACD,EAAE,EACF,EAAE,KAAK,EAAE,OAAO,EAAE,CACnB,CAAC;IACJ,CAAC;IACD,aAAa,CAAC,IAAY,EAAE,KAAK,GAAG,KAAK;QACvC,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CACvB,mCAAmC,kBAAkB,CAAC,IAAI,CAAC,EAAE,EAC7D,SAAS,EACT,EAAE,KAAK,EAAE,EACT,EAAE,KAAK,EAAE,OAAO,EAAE,CACnB,CAAC;IACJ,CAAC;IACD,YAAY,CAAC,QAAgB,EAAE,KAAK,GAAG,KAAK;QAC1C,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CACvB,kCAAkC,kBAAkB,CAAC,QAAQ,CAAC,EAAE,EAChE,SAAS,EACT,EAAE,KAAK,EAAE,EACT,EAAE,KAAK,EAAE,OAAO,EAAE,CACnB,CAAC;IACJ,CAAC;IACD,YAAY,CAAC,QAAgB;QAC3B,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CACvB,kCAAkC,kBAAkB,CAAC,QAAQ,CAAC,EAAE,EAChE,SAAS,EACT,EAAE,EACF,EAAE,KAAK,EAAE,OAAO,EAAE,CACnB,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,aAAa,CAAC,IAAqC,EAAE,QAAgB;QACzE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAC/B,0BAA0B,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,IAAI,kBAAkB,CAAC,QAAQ,CAAC,EAAE,CACxI,CAAC;QACF,MAAM,UAAU,GAAG,GAAG,CAAC,UAAiD,CAAC;QACzE,MAAM,OAAO,GACX,IAAI,KAAK,QAAQ;YACf,CAAC,CAAC,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,0BAA0B,CAAC;YACvF,CAAC,CAAC,UAAU,EAAE,CAAC,gBAAgB,CAAC,KAAK,MAAM,CAAC;QAChD,IAAI,CAAC,OAAO;YACV,MAAM,IAAI,KAAK,CAAC,+CAA+C,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC;QACxF,IAAI,IAAI,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACnE,IAAI,IAAI,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACjE,OAAO,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC;CACF;AAED,MAAM,UAAU,gCAAgC,CAC9C,MAA4B,EAC5B,UAAqC,EAAE;IAEvC,MAAM,GAAG,GAAG,IAAI,2BAA2B,CAAC,MAAM,CAAC,CAAC;IACpD,MAAM,KAAK,GAAc;QACvB,IAAI,CACF,kBAAkB,EAClB,OAAO,EACP,gBAAgB,EAAE,EAClB,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,WAAW,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,EAClE,CAAC,CAAC,EAAE,EAAE,CACJ,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,aAAwC,EAAE,CAAC,CAAC,QAAkB,EAAE;YACtF,EAAE,SAAS,EAAE,CAAC,CAAC,SAAmB,EAAE,GAAG,EAAE,CAAC,CAAC,UAAsB,EAAE;SACpE,CAAC,CACL;QACD,IAAI,CACF,mBAAmB,EACnB,OAAO,EACP,gBAAgB,EAAE,EAClB,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,WAAW,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,EAClE,CAAC,CAAC,EAAE,EAAE,CACJ,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,aAAwC,EAAE,CAAC,CAAC,QAAkB,EAAE;YACtF,EAAE,SAAS,EAAE,CAAC,CAAC,SAAmB,EAAE,MAAM,EAAE,CAAC,CAAC,UAAsB,EAAE;SACvE,CAAC,CACL;QACD,IAAI,CACF,2BAA2B,EAC3B,OAAO,EACP,aAAa,EAAE,EACf,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,EAC5D,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,aAAa,CAAC,CAAU,CAAC,CACrC;QACD,IAAI,CACF,0BAA0B,EAC1B,OAAO,EACP,YAAY,EAAE,EACd;YACE,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,cAAc,EAAE,gBAAgB,EAAE,IAAI,EAAE;YACjE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE;SAClC,EACD,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,YAAY,CAAC,CAAU,CAAC,CACpC;QACD,IAAI,CACF,0BAA0B,EAC1B,OAAO,EACP,YAAY,EAAE,EACd;YACE,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,cAAc,EAAE,gBAAgB,EAAE,IAAI,EAAE;YACjE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,aAAa,EAAE;YACvC,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE;SAClC,EACD,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,YAAY,CAAC,CAAU,CAAC,CACpC;KACF,CAAC;IACF,IAAI,OAAO,CAAC,WAAW;QACrB,KAAK,CAAC,IAAI,CACR,IAAI,CACF,6BAA6B,EAC7B,OAAO,EACP,YAAY,EAAE,EACd,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,WAAW,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,EAClE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,IAAa,EAAE,CAAC,CAAC,QAAkB,CAAC,CAChE,CACF,CAAC;IACJ,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,IAAI,CACX,IAAY,EACZ,MAAe,EACf,WAAuB,EACvB,SAAuB,EACvB,OAAuC;IAEvC,OAAO;QACL,IAAI;QACJ,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC;QACtC,WAAW;QACX,QAAQ,EAAE;YACR,MAAM;YACN,QAAQ,EAAE,YAAY;YACtB,OAAO,EAAE,eAAe;YACxB,UAAU,EAAE,EAAE,SAAS,EAAE;SAC1B;QACD,OAAO;KACR,CAAC;AACJ,CAAC;AACD,SAAS,gBAAgB;IACvB,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,oBAAoB,EAAE,KAAK;QAC3B,UAAU,EAAE;YACV,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACjC,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC5B,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC7B,UAAU,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;SACzD;QACD,QAAQ,EAAE,CAAC,eAAe,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,CAAC;KACnE,CAAC;AACJ,CAAC;AACD,SAAS,aAAa;IACpB,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,oBAAoB,EAAE,KAAK;QAC3B,UAAU,EAAE;YACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACxB,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC3B,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC/B,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,oBAAoB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;SACzE;QACD,QAAQ,EAAE,CAAC,MAAM,CAAC;KACnB,CAAC;AACJ,CAAC;AACD,SAAS,YAAY;IACnB,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,oBAAoB,EAAE,KAAK;QAC3B,UAAU,EAAE;YACV,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC/B,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACxB,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC3B,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,oBAAoB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;SACzE;QACD,QAAQ,EAAE,CAAC,aAAa,EAAE,MAAM,CAAC;KAClC,CAAC;AACJ,CAAC;AACD,SAAS,YAAY;IACnB,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,oBAAoB,EAAE,KAAK;QAC3B,UAAU,EAAE;YACV,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC/B,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC9B,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACxB,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,SAAS,EAAE,UAAU,CAAC,EAAE;YAC7D,eAAe,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACnC,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;SAC5B;QACD,QAAQ,EAAE,CAAC,aAAa,EAAE,YAAY,EAAE,MAAM,CAAC;KAChD,CAAC;AACJ,CAAC;AACD,SAAS,YAAY;IACnB,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,oBAAoB,EAAE,KAAK;QAC3B,UAAU,EAAE;YACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,SAAS,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE;YAC/D,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;SAC7B;QACD,QAAQ,EAAE,CAAC,MAAM,EAAE,UAAU,CAAC;KAC/B,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
const SCORE_COLUMN = 'score';
|
|
2
|
+
/** Databricks AI Search as a {@link Retriever}. Runs under the bundle's Unity Catalog principal. */
|
|
3
|
+
export function databricksAiSearch(options) {
|
|
4
|
+
const queryType = options.queryType ?? 'text';
|
|
5
|
+
const columns = options.columns ?? dedupe([options.textColumn, ...(options.idColumn ? [options.idColumn] : [])]);
|
|
6
|
+
return {
|
|
7
|
+
name: options.name ?? `databricks-ai-search:${options.index}`,
|
|
8
|
+
async retrieve(query, opts = {}) {
|
|
9
|
+
const body = { columns, num_results: opts.k ?? 5 };
|
|
10
|
+
if (opts.filter)
|
|
11
|
+
body.filters_json = JSON.stringify(opts.filter);
|
|
12
|
+
if (queryType === 'vector') {
|
|
13
|
+
if (!options.embeddings)
|
|
14
|
+
throw new Error('databricks AI Search: queryType "vector" requires an embeddings provider.');
|
|
15
|
+
const [vector] = await options.embeddings.embed([query]);
|
|
16
|
+
if (!vector)
|
|
17
|
+
throw new Error('databricks AI Search: embeddings returned no vector for the query.');
|
|
18
|
+
body.query_vector = vector;
|
|
19
|
+
}
|
|
20
|
+
else {
|
|
21
|
+
body.query_text = query;
|
|
22
|
+
}
|
|
23
|
+
const response = await options.client.post(`/api/2.0/vector-search/indexes/${encodeURIComponent(options.index)}/query`, body);
|
|
24
|
+
return mapResults(response, options, columns);
|
|
25
|
+
},
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
/** @deprecated Use {@link databricksAiSearch}. */
|
|
29
|
+
export const databricksVectorSearch = databricksAiSearch;
|
|
30
|
+
function mapResults(response, options, requestedColumns) {
|
|
31
|
+
const names = (response.manifest?.columns ?? []).map((c) => c.name);
|
|
32
|
+
const requested = new Set(requestedColumns);
|
|
33
|
+
// Identify the score column: a literal `score`, else a single trailing column the response
|
|
34
|
+
// appended beyond what we requested (Databricks adds the similarity score). Never treat a
|
|
35
|
+
// requested column as the score — that would mis-attribute a numeric metadata field.
|
|
36
|
+
const lastName = names[names.length - 1];
|
|
37
|
+
const scoreColumn = names.includes(SCORE_COLUMN)
|
|
38
|
+
? SCORE_COLUMN
|
|
39
|
+
: (lastName && !requested.has(lastName) ? lastName : undefined);
|
|
40
|
+
const rows = response.result?.data_array ?? [];
|
|
41
|
+
return rows.map((row) => {
|
|
42
|
+
const record = {};
|
|
43
|
+
names.forEach((name, i) => { record[name] = row[i]; });
|
|
44
|
+
const text = String(record[options.textColumn] ?? '');
|
|
45
|
+
const scoreRaw = scoreColumn ? record[scoreColumn] : undefined;
|
|
46
|
+
const score = typeof scoreRaw === 'number' ? scoreRaw : undefined;
|
|
47
|
+
const idRaw = options.idColumn ? record[options.idColumn] : undefined;
|
|
48
|
+
const metadata = { ...record };
|
|
49
|
+
delete metadata[options.textColumn];
|
|
50
|
+
if (scoreColumn)
|
|
51
|
+
delete metadata[scoreColumn];
|
|
52
|
+
if (options.idColumn)
|
|
53
|
+
delete metadata[options.idColumn];
|
|
54
|
+
return {
|
|
55
|
+
...(idRaw != null ? { id: String(idRaw) } : {}),
|
|
56
|
+
text,
|
|
57
|
+
...(score !== undefined ? { score } : {}),
|
|
58
|
+
metadata,
|
|
59
|
+
};
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
function dedupe(values) {
|
|
63
|
+
return [...new Set(values)];
|
|
64
|
+
}
|
|
65
|
+
//# sourceMappingURL=vector-search.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vector-search.js","sourceRoot":"","sources":["../src/vector-search.ts"],"names":[],"mappings":"AA4BA,MAAM,YAAY,GAAG,OAAO,CAAC;AAE7B,oGAAoG;AACpG,MAAM,UAAU,kBAAkB,CAAC,OAAkC;IACnE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC;IAC9C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,MAAM,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAEjH,OAAO;QACL,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,wBAAwB,OAAO,CAAC,KAAK,EAAE;QAC7D,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAwB,EAAE;YACtD,MAAM,IAAI,GAAe,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/D,IAAI,IAAI,CAAC,MAAM;gBAAE,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACjE,IAAI,SAAS,KAAK,QAAQ,EAAE,CAAC;gBAC3B,IAAI,CAAC,OAAO,CAAC,UAAU;oBAAE,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;gBACtH,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;gBACzD,IAAI,CAAC,MAAM;oBAAE,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;gBACnG,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;YAC1B,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,IAAI,CACxC,kCAAkC,kBAAkB,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,EAC3E,IAAI,CACL,CAAC;YACF,OAAO,UAAU,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAChD,CAAC;KACF,CAAC;AACJ,CAAC;AAKD,kDAAkD;AAClD,MAAM,CAAC,MAAM,sBAAsB,GAAG,kBAAkB,CAAC;AAEzD,SAAS,UAAU,CAAC,QAAmC,EAAE,OAAkC,EAAE,gBAA0B;IACrH,MAAM,KAAK,GAAG,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACpE,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC5C,2FAA2F;IAC3F,0FAA0F;IAC1F,qFAAqF;IACrF,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACzC,MAAM,WAAW,GAAG,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC;QAC9C,CAAC,CAAC,YAAY;QACd,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAElE,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,EAAE,UAAU,IAAI,EAAE,CAAC;IAC/C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACtB,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,CAAuB,CAAC,CAAC,CAAC,CAAC,CAAC;QAE7E,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;QACtD,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC/D,MAAM,KAAK,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;QAClE,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAEtE,MAAM,QAAQ,GAAe,EAAE,GAAG,MAAM,EAAE,CAAC;QAC3C,OAAO,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACpC,IAAI,WAAW;YAAE,OAAO,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC9C,IAAI,OAAO,CAAC,QAAQ;YAAE,OAAO,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAExD,OAAO;YACL,GAAG,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/C,IAAI;YACJ,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzC,QAAQ;SACT,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,MAAM,CAAC,MAAgB;IAC9B,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;AAC9B,CAAC"}
|