@fabric-harness/databricks 0.5.0 → 0.6.0

package/README.md

@@ -2,7 +2,7 @@
 
 Databricks integration helpers for Fabric Harness.
 
-Current surface:
+## Current surface
 
 - `DatabricksRestClient` — small token-authenticated REST client.
 - `databricksSqlTool()` — run SQL Warehouse statements from an agent tool.
@@ -11,5 +11,53 @@ Current surface:
 - `unityCatalogTablesTool()` — list Unity Catalog tables.
 - `databricksMlflowLogMetricTool()` / `databricksMlflowLogParamTool()` — write MLflow telemetry from an agent run.
 - `databricksWorkspaceSource()` — mount exported workspace notebooks/files as a Fabric filesystem source.
+- **`databricksSqlSandbox()`** (subpath: `@fabric-harness/databricks/sql-sandbox`) — full `SandboxEnv` whose `exec(sql)` runs against a SQL Warehouse, with results serialized as JSONL or CSV.
+
+## Install
+
+```sh
+npm install @fabric-harness/databricks @fabric-harness/sdk
+```
+
+## SQL Warehouse sandbox
+
+Run an entire agent against a SQL Warehouse, with `session.shell()` (or `session.prompt()` tool calls) routed to SQL statement execution:
+
+```ts
+import { init } from '@fabric-harness/sdk';
+import { databricksSqlSandbox } from '@fabric-harness/databricks/sql-sandbox';
+
+const fabric = await init({
+  sandbox: databricksSqlSandbox({
+    host: process.env.DATABRICKS_HOST!,
+    token: process.env.DATABRICKS_TOKEN!,
+    warehouseId: process.env.DATABRICKS_WAREHOUSE_ID!,
+    catalog: 'main',
+    schema: 'analytics',
+    resultFormat: 'jsonl', // or 'csv'
+  }),
+});
+
+const session = await fabric.session();
+const result = await session.shell('SELECT customer_id, SUM(amount) FROM orders GROUP BY 1');
+console.log(result.stdout); // newline-delimited JSON rows
+```
+
+For real data files, mount with `databricksVolumeSource` from `@fabric-harness/connectors/databricks-volume`:
+
+```ts
+import { databricksVolumeSource } from '@fabric-harness/connectors/databricks-volume';
+
+await session.mount('/mnt/landing', databricksVolumeSource({
+  host: process.env.DATABRICKS_HOST!,
+  token: process.env.DATABRICKS_TOKEN!,
+  volumePath: '/Volumes/main/landing/raw',
+}));
+```
+
+## Documentation
+
+- [Databricks deployment guide](https://harness.fabric.pro/docs/deployment/databricks)
+- [Roadmap](https://github.com/Fabric-Pro/fabric-harness/blob/main/docs/ROADMAP.md) — cluster/notebook-backed `SandboxEnv` and Jobs deploy target are planned.
 
 Keep Databricks hostnames and tokens in environment variables or secret stores; do not put them in prompts or payloads.

package/dist/sql-sandbox.d.ts

@@ -0,0 +1,48 @@
+import type { SandboxEnv } from '@fabric-harness/sdk';
+import { DatabricksRestClient } from './index.js';
+/**
+ * Sandbox backend that maps `exec(command)` to a Databricks SQL Statement
+ * Execution API call against a SQL Warehouse. Useful for data agents whose
+ * "shell" is a query interface.
+ *
+ * Behavior:
+ * - `exec(sql)` runs the SQL synchronously (with a timeout) and returns the
+ *   serialized result set as `stdout` (newline-delimited JSON rows by default,
+ *   or CSV when `resultFormat: 'csv'`).
+ * - File operations are intentionally minimal: the sandbox-virtual filesystem
+ *   keeps a small in-memory map. SQL warehouses are not file servers; mount
+ *   actual data via `databricksVolumeSource` from `@fabric-harness/connectors`.
+ *
+ * Usage:
+ * ```ts
+ * import { databricksSqlSandbox } from '@fabric-harness/databricks/sql-sandbox';
+ *
+ * const fabric = await init({
+ *   sandbox: databricksSqlSandbox({
+ *     host: process.env.DATABRICKS_HOST!,
+ *     token: process.env.DATABRICKS_TOKEN!,
+ *     warehouseId: process.env.DATABRICKS_WAREHOUSE_ID!,
+ *   }),
+ * });
+ * ```
+ */
+export interface DatabricksSqlSandboxOptions {
+    host: string;
+    token: string | (() => string | Promise<string>);
+    /** SQL Warehouse ID to execute against. */
+    warehouseId: string;
+    /** Optional Unity Catalog. */
+    catalog?: string;
+    /** Optional schema. */
+    schema?: string;
+    /** Result row serialization. Defaults to `'jsonl'`. */
+    resultFormat?: 'jsonl' | 'csv';
+    /** Wait timeout for the statement, in seconds (10–50). Defaults to `30`. */
+    waitTimeoutSeconds?: number;
+    /** Custom fetch implementation (testing, OBO tokens, etc.). */
+    fetchImpl?: typeof fetch;
+    /** Inject a pre-built REST client (escape hatch for tests). */
+    client?: DatabricksRestClient;
+}
+export declare function databricksSqlSandbox(options: DatabricksSqlSandboxOptions): SandboxEnv;
+//# sourceMappingURL=sql-sandbox.d.ts.map

package/dist/sql-sandbox.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sql-sandbox.d.ts","sourceRoot":"","sources":["../src/sql-sandbox.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAoB,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACxE,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAElD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,WAAW,2BAA2B;IAC1C,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,GAAG,CAAC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IACjD,2CAA2C;IAC3C,WAAW,EAAE,MAAM,CAAC;IACpB,8BAA8B;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uBAAuB;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,uDAAuD;IACvD,YAAY,CAAC,EAAE,OAAO,GAAG,KAAK,CAAC;IAC/B,4EAA4E;IAC5E,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,+DAA+D;IAC/D,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,+DAA+D;IAC/D,MAAM,CAAC,EAAE,oBAAoB,CAAC;CAC/B;AASD,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,2BAA2B,GAAG,UAAU,CAsFrF"}

package/dist/sql-sandbox.js

@@ -0,0 +1,116 @@
+import { createRemoteSandboxEnv } from '@fabric-harness/sdk';
+import { DatabricksRestClient } from './index.js';
+export function databricksSqlSandbox(options) {
+    const client = options.client ?? new DatabricksRestClient({
+        host: options.host,
+        token: options.token,
+        ...(options.fetchImpl ? { fetchImpl: options.fetchImpl } : {}),
+    });
+    const waitTimeoutSeconds = options.waitTimeoutSeconds ?? 30;
+    const resultFormat = options.resultFormat ?? 'jsonl';
+    const memFs = new Map();
+    const api = {
+        async exec(command) {
+            const startedAt = Date.now();
+            try {
+                const response = await client.post('/api/2.0/sql/statements', {
+                    warehouse_id: options.warehouseId,
+                    statement: command,
+                    wait_timeout: `${waitTimeoutSeconds}s`,
+                    on_wait_timeout: 'CANCEL',
+                    ...(options.catalog ? { catalog: options.catalog } : {}),
+                    ...(options.schema ? { schema: options.schema } : {}),
+                });
+                if (response.status.state === 'FAILED' || response.status.state === 'CANCELED') {
+                    return {
+                        command,
+                        exitCode: 1,
+                        stdout: '',
+                        stderr: response.status.error?.message ?? `Statement ${response.status.state}`,
+                        durationMs: Date.now() - startedAt,
+                    };
+                }
+                const stdout = serializeResult(response, resultFormat);
+                return { command, exitCode: 0, stdout, stderr: '', durationMs: Date.now() - startedAt };
+            }
+            catch (error) {
+                return {
+                    command,
+                    exitCode: 1,
+                    stdout: '',
+                    stderr: error instanceof Error ? error.message : String(error),
+                    durationMs: Date.now() - startedAt,
+                };
+            }
+        },
+        async readFile(path) {
+            const bytes = memFs.get(path);
+            if (!bytes)
+                throw new Error(`No such file: ${path}`);
+            return new TextDecoder().decode(bytes);
+        },
+        async readFileBuffer(path) {
+            const bytes = memFs.get(path);
+            if (!bytes)
+                throw new Error(`No such file: ${path}`);
+            return bytes;
+        },
+        async writeFile(path, content) {
+            memFs.set(path, typeof content === 'string' ? new TextEncoder().encode(content) : content);
+        },
+        async stat(path) {
+            const bytes = memFs.get(path);
+            if (!bytes)
+                throw new Error(`No such file: ${path}`);
+            return { type: 'file', size: bytes.byteLength };
+        },
+        async readdir(path) {
+            const prefix = path.replace(/\/$/, '');
+            const direct = new Set();
+            for (const key of memFs.keys()) {
+                if (!key.startsWith(`${prefix}/`))
+                    continue;
+                const tail = key.slice(prefix.length + 1);
+                const slash = tail.indexOf('/');
+                direct.add(slash > 0 ? tail.slice(0, slash) : tail);
+            }
+            return [...direct];
+        },
+        async exists(path) {
+            return memFs.has(path);
+        },
+        async mkdir() { },
+        async rm(path) {
+            for (const key of [...memFs.keys()]) {
+                if (key === path || key.startsWith(`${path}/`))
+                    memFs.delete(key);
+            }
+        },
+    };
+    return createRemoteSandboxEnv(api, {
+        cwd: '/workspace',
+        capabilities: { exec: true, filesystem: true, binaryFiles: true, snapshots: false, restore: false, network: 'provider-managed', persistence: 'session', isolation: 'provider' },
+    });
+}
+function serializeResult(response, format) {
+    const columns = response.manifest?.schema?.columns?.map((c) => c.name ?? '') ?? [];
+    const rows = response.result?.data_array ?? [];
+    if (format === 'csv') {
+        const header = columns.join(',');
+        const body = rows.map((row) => row.map(formatCsvCell).join(',')).join('\n');
+        return [header, body].filter(Boolean).join('\n');
+    }
+    return rows.map((row) => {
+        const obj = {};
+        for (let i = 0; i < columns.length; i += 1)
+            obj[columns[i] ?? `_${i}`] = row[i];
+        return JSON.stringify(obj);
+    }).join('\n');
+}
+function formatCsvCell(value) {
+    if (value === null || value === undefined)
+        return '';
+    const str = String(value);
+    return /[",\n]/.test(str) ? `"${str.replace(/"/g, '""')}"` : str;
+}
+//# sourceMappingURL=sql-sandbox.js.map

package/dist/sql-sandbox.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sql-sandbox.js","sourceRoot":"","sources":["../src/sql-sandbox.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAE7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAsDlD,MAAM,UAAU,oBAAoB,CAAC,OAAoC;IACvE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,IAAI,oBAAoB,CAAC;QACxD,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC/D,CAAC,CAAC;IACH,MAAM,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAC;IAC5D,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC;IACrD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAsB,CAAC;IAE5C,MAAM,GAAG,GAAqB;QAC5B,KAAK,CAAC,IAAI,CAAC,OAAO;YAChB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAoB,yBAAyB,EAAE;oBAC/E,YAAY,EAAE,OAAO,CAAC,WAAW;oBACjC,SAAS,EAAE,OAAO;oBAClB,YAAY,EAAE,GAAG,kBAAkB,GAAG;oBACtC,eAAe,EAAE,QAAQ;oBACzB,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACxD,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACtD,CAAC,CAAC;gBACH,IAAI,QAAQ,CAAC,MAAM,CAAC,KAAK,KAAK,QAAQ,IAAI,QAAQ,CAAC,MAAM,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;oBAC/E,OAAO;wBACL,OAAO;wBACP,QAAQ,EAAE,CAAC;wBACX,MAAM,EAAE,EAAE;wBACV,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,IAAI,aAAa,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE;wBAC9E,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;qBACnC,CAAC;gBACJ,CAAC;gBACD,MAAM,MAAM,GAAG,eAAe,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;gBACvD,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,CAAC;YAC1F,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO;oBACP,QAAQ,EAAE,CAAC;oBACX,MAAM,EAAE,EAAE;oBACV,MAAM,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;oBAC9D,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;iBACnC,CAAC;YACJ,CAAC;QACH,CAAC;QACD,KAAK,CAAC,QAAQ,CAAC,IAAI;YACjB,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC9B,IAAI,CAAC,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC;YACrD,OAAO,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACzC,CAAC;QACD,KAAK,CAAC,cAAc,CAAC,IAAI;YACvB,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC9B,IAAI,CAAC,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC;YACrD,OAAO,KAAK,CAAC;QACf,CAAC;QACD,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,OAAO;YAC3B,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC7F,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,IAAI;YACb,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC9B,IAAI,CAAC,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC;YACrD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC;QAClD,CAAC;QACD,KAAK,CAAC,OAAO,CAAC,IAAI;YAChB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACvC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;YACjC,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC/B,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,MAAM,GAAG,CAAC;oBAAE,SAAS;gBAC5C,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBAChC,MAAM,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACtD,CAAC;YACD,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC;QACrB,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,IAAI;YACf,OAAO,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACzB,CAAC;QACD,KAAK,CAAC,KAAK,KAA0D,CAAC;QACtE,KAAK,CAAC,EAAE,CAAC,IAAI;YACX,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;gBACpC,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,CAAC;oBAAE,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACpE,CAAC;QACH,CAAC;KACF,CAAC;IACF,OAAO,sBAAsB,CAAC,GAAG,EAAE;QACjC,GAAG,EAAE,YAAY;QACjB,YAAY,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE;KAChL,CAAC,CAAC;AACL,CAAC;AAED,SAAS,eAAe,CAAC,QAA2B,EAAE,MAAuB;IAC3E,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IACnF,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,EAAE,UAAU,IAAI,EAAE,CAAC;IAC/C,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5E,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACtB,MAAM,GAAG,GAA4B,EAAE,CAAC;QACxC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC;YAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAChF,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IACrD,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1B,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;AACnE,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fabric-harness/databricks",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "Databricks tools and filesystem sources for Fabric Harness agents.",
   "license": "Apache-2.0",
   "author": "Fabric",
@@ -32,13 +32,23 @@
   },
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./sql-sandbox": {
+      "types": "./dist/sql-sandbox.d.ts",
+      "import": "./dist/sql-sandbox.js"
+    }
+  },
   "files": [
     "dist",
     "README.md",
     "LICENSE"
   ],
   "dependencies": {
-    "@fabric-harness/sdk": "0.5.0"
+    "@fabric-harness/sdk": "0.6.0"
   },
   "devDependencies": {
     "typescript": "^5.8.3",