dataiku-sdk 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/dss.js +3 -1
- package/dist/packages/types/src/index.d.ts +4 -4
- package/dist/packages/types/src/index.js +2 -2
- package/dist/src/auth.d.ts +4 -0
- package/dist/src/auth.js +20 -0
- package/dist/src/cli.js +515 -72
- package/dist/src/client.d.ts +5 -1
- package/dist/src/client.js +14 -1
- package/dist/src/config.d.ts +11 -0
- package/dist/src/config.js +64 -0
- package/dist/src/errors.js +12 -0
- package/dist/src/index.d.ts +2 -0
- package/dist/src/index.js +3 -0
- package/dist/src/resources/connections.d.ts +1 -1
- package/dist/src/resources/connections.js +7 -4
- package/dist/src/resources/datasets.js +8 -2
- package/dist/src/resources/folders.d.ts +1 -0
- package/dist/src/resources/folders.js +8 -0
- package/dist/src/resources/jobs.d.ts +2 -1
- package/dist/src/resources/jobs.js +7 -3
- package/dist/src/resources/notebooks.d.ts +18 -3
- package/dist/src/resources/notebooks.js +25 -5
- package/dist/src/resources/recipes.d.ts +18 -1
- package/dist/src/resources/recipes.js +68 -7
- package/dist/src/resources/sql.js +32 -5
- package/dist/src/resources/variables.d.ts +1 -0
- package/dist/src/resources/variables.js +9 -1
- package/dist/src/skill.d.ts +33 -0
- package/dist/src/skill.js +229 -0
- package/package.json +1 -1
- package/packages/types/dist/index.d.ts +4 -4
- package/packages/types/dist/index.js +2 -2
package/dist/src/client.d.ts
CHANGED
|
@@ -21,9 +21,11 @@ export interface DataikuClientConfig {
|
|
|
21
21
|
requestTimeoutMs?: number;
|
|
22
22
|
/** Max retry attempts for idempotent requests (default 4, capped at 10) */
|
|
23
23
|
retryMaxAttempts?: number;
|
|
24
|
+
/** Emit HTTP request/response logs to stderr for CLI debugging. */
|
|
25
|
+
verbose?: boolean;
|
|
24
26
|
/**
|
|
25
27
|
* Called when an API response fails schema validation but data is still usable.
|
|
26
|
-
* Default:
|
|
28
|
+
* Default: writes to stderr. Set to a throwing function for strict mode.
|
|
27
29
|
* @param method - resource method that triggered the warning (e.g. "datasets.list")
|
|
28
30
|
* @param errors - human-readable validation error strings
|
|
29
31
|
*/
|
|
@@ -35,6 +37,7 @@ export declare class DataikuClient {
|
|
|
35
37
|
private readonly defaultProjectKey;
|
|
36
38
|
private readonly requestTimeoutMs;
|
|
37
39
|
private readonly retryMaxAttempts;
|
|
40
|
+
private readonly verbose;
|
|
38
41
|
private readonly onValidationWarning;
|
|
39
42
|
private _projects?;
|
|
40
43
|
private _datasets?;
|
|
@@ -70,6 +73,7 @@ export declare class DataikuClient {
|
|
|
70
73
|
stream(path: string): Promise<Response>;
|
|
71
74
|
private getHeaders;
|
|
72
75
|
private getAnyHeaders;
|
|
76
|
+
private logVerbose;
|
|
73
77
|
/**
|
|
74
78
|
* Validate raw data against a TypeBox schema, throwing on structural mismatch.
|
|
75
79
|
* Resources call this instead of bare `as T` casts for validated responses.
|
package/dist/src/client.js
CHANGED
|
@@ -24,7 +24,7 @@ const DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
|
|
|
24
24
|
/* Helpers */
|
|
25
25
|
/* ------------------------------------------------------------------ */
|
|
26
26
|
function defaultValidationWarning(method, errors) {
|
|
27
|
-
|
|
27
|
+
process.stderr.write(`[dataiku-sdk] Schema validation warning in ${method}:\n ${errors.join("\n ")}\n`);
|
|
28
28
|
}
|
|
29
29
|
function sleep(ms) {
|
|
30
30
|
return new Promise((r) => setTimeout(r, ms));
|
|
@@ -59,6 +59,7 @@ export class DataikuClient {
|
|
|
59
59
|
defaultProjectKey;
|
|
60
60
|
requestTimeoutMs;
|
|
61
61
|
retryMaxAttempts;
|
|
62
|
+
verbose;
|
|
62
63
|
onValidationWarning;
|
|
63
64
|
/* Resource namespaces — lazily initialized to break circular imports */
|
|
64
65
|
_projects;
|
|
@@ -118,6 +119,7 @@ export class DataikuClient {
|
|
|
118
119
|
this.requestTimeoutMs = config.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
|
|
119
120
|
const rawMax = config.retryMaxAttempts ?? DEFAULT_RETRY_MAX_ATTEMPTS;
|
|
120
121
|
this.retryMaxAttempts = Math.min(Math.max(1, rawMax), MAX_RETRY_ATTEMPTS_CAP);
|
|
122
|
+
this.verbose = config.verbose === true;
|
|
121
123
|
this.onValidationWarning = config.onValidationWarning ?? defaultValidationWarning;
|
|
122
124
|
}
|
|
123
125
|
/* ---- public: project key resolution ---- */
|
|
@@ -206,6 +208,10 @@ export class DataikuClient {
|
|
|
206
208
|
Accept: "*/*",
|
|
207
209
|
};
|
|
208
210
|
}
|
|
211
|
+
logVerbose(message) {
|
|
212
|
+
if (this.verbose)
|
|
213
|
+
process.stderr.write(`[dss] ${message}\n`);
|
|
214
|
+
}
|
|
209
215
|
/* ---- public: schema-validated parsing ---- */
|
|
210
216
|
/**
|
|
211
217
|
* Validate raw data against a TypeBox schema, throwing on structural mismatch.
|
|
@@ -235,6 +241,9 @@ export class DataikuClient {
|
|
|
235
241
|
/* ---- private: JSON parsing ---- */
|
|
236
242
|
async parseJsonResponse(res) {
|
|
237
243
|
const text = await res.text();
|
|
244
|
+
// SAFETY: Empty 2xx responses from DSS are surfaced to callers as undefined
|
|
245
|
+
// cast to T. This keeps existing call sites stable, but callers that rely on
|
|
246
|
+
// an object shape must guard explicitly before dereferencing the result.
|
|
238
247
|
if (!text)
|
|
239
248
|
return undefined;
|
|
240
249
|
try {
|
|
@@ -253,13 +262,16 @@ export class DataikuClient {
|
|
|
253
262
|
const delaysMs = [];
|
|
254
263
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
255
264
|
let timedOut = false;
|
|
265
|
+
const startedAt = Date.now();
|
|
256
266
|
const controller = new AbortController();
|
|
257
267
|
const timeout = setTimeout(() => {
|
|
258
268
|
timedOut = true;
|
|
259
269
|
controller.abort();
|
|
260
270
|
}, this.requestTimeoutMs);
|
|
271
|
+
this.logVerbose(`${method} ${url}`);
|
|
261
272
|
try {
|
|
262
273
|
const res = await fetch(url, { ...init, method, signal: controller.signal, });
|
|
274
|
+
this.logVerbose(`${method} ${url} → ${res.status} (${Date.now() - startedAt}ms)`);
|
|
263
275
|
if (!res.ok) {
|
|
264
276
|
const text = await res.text();
|
|
265
277
|
const canRetry = retryEnabled && attempt < maxAttempts && isTransientError(res.status, text);
|
|
@@ -288,6 +300,7 @@ export class DataikuClient {
|
|
|
288
300
|
: error instanceof Error
|
|
289
301
|
? error.message
|
|
290
302
|
: "Unknown transport error";
|
|
303
|
+
this.logVerbose(`${method} ${url} → ERROR (${Date.now() - startedAt}ms) ${detail}`);
|
|
291
304
|
const statusText = timedOut ? "Request Timeout" : "Network Error";
|
|
292
305
|
throw new DataikuError(0, statusText, detail, buildRetryMetadata(method, retryEnabled, maxAttempts, attempt, delaysMs, timedOut));
|
|
293
306
|
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export interface DssCredentials {
|
|
2
|
+
url: string;
|
|
3
|
+
apiKey: string;
|
|
4
|
+
projectKey?: string;
|
|
5
|
+
}
|
|
6
|
+
export declare function getConfigDir(): string;
|
|
7
|
+
export declare function getCredentialsPath(): string;
|
|
8
|
+
export declare function loadCredentials(): DssCredentials | null;
|
|
9
|
+
export declare function saveCredentials(creds: DssCredentials): void;
|
|
10
|
+
export declare function deleteCredentials(): void;
|
|
11
|
+
export declare function maskApiKey(apiKey: string): string;
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { chmodSync, mkdirSync, readFileSync, unlinkSync, writeFileSync, } from "node:fs";
|
|
2
|
+
import { homedir, } from "node:os";
|
|
3
|
+
import { dirname, join, resolve, } from "node:path";
|
|
4
|
+
export function getConfigDir() {
|
|
5
|
+
if (process.env.DSS_CONFIG_DIR)
|
|
6
|
+
return process.env.DSS_CONFIG_DIR;
|
|
7
|
+
if (process.env.XDG_CONFIG_HOME)
|
|
8
|
+
return resolve(process.env.XDG_CONFIG_HOME, "dataiku");
|
|
9
|
+
if (process.platform === "win32" && process.env.APPDATA) {
|
|
10
|
+
return resolve(process.env.APPDATA, "dataiku");
|
|
11
|
+
}
|
|
12
|
+
return resolve(homedir(), ".config", "dataiku");
|
|
13
|
+
}
|
|
14
|
+
export function getCredentialsPath() {
|
|
15
|
+
return join(getConfigDir(), "credentials.json");
|
|
16
|
+
}
|
|
17
|
+
export function loadCredentials() {
|
|
18
|
+
try {
|
|
19
|
+
const raw = readFileSync(getCredentialsPath(), "utf-8");
|
|
20
|
+
const parsed = JSON.parse(raw);
|
|
21
|
+
if (!parsed
|
|
22
|
+
|| typeof parsed !== "object"
|
|
23
|
+
|| Array.isArray(parsed)
|
|
24
|
+
|| typeof parsed.url !== "string"
|
|
25
|
+
|| typeof parsed.apiKey !== "string") {
|
|
26
|
+
return null;
|
|
27
|
+
}
|
|
28
|
+
const obj = parsed;
|
|
29
|
+
return {
|
|
30
|
+
url: obj.url,
|
|
31
|
+
apiKey: obj.apiKey,
|
|
32
|
+
projectKey: typeof obj.projectKey === "string" ? obj.projectKey : undefined,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
catch (err) {
|
|
36
|
+
if (err.code === "ENOENT")
|
|
37
|
+
return null;
|
|
38
|
+
throw err;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
export function saveCredentials(creds) {
|
|
42
|
+
const path = getCredentialsPath();
|
|
43
|
+
mkdirSync(dirname(path), { recursive: true, });
|
|
44
|
+
const data = { url: creds.url, apiKey: creds.apiKey, };
|
|
45
|
+
if (creds.projectKey)
|
|
46
|
+
data.projectKey = creds.projectKey;
|
|
47
|
+
writeFileSync(path, `${JSON.stringify(data, null, 2)}\n`, "utf-8");
|
|
48
|
+
chmodSync(path, 0o600);
|
|
49
|
+
}
|
|
50
|
+
export function deleteCredentials() {
|
|
51
|
+
try {
|
|
52
|
+
unlinkSync(getCredentialsPath());
|
|
53
|
+
}
|
|
54
|
+
catch (err) {
|
|
55
|
+
if (err.code === "ENOENT")
|
|
56
|
+
return;
|
|
57
|
+
throw err;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
export function maskApiKey(apiKey) {
|
|
61
|
+
if (apiKey.length <= 12)
|
|
62
|
+
return "***";
|
|
63
|
+
return `${apiKey.slice(0, 6)}...${apiKey.slice(-6)}`;
|
|
64
|
+
}
|
package/dist/src/errors.js
CHANGED
|
@@ -39,6 +39,18 @@ export function classifyDataikuError(status, body) {
|
|
|
39
39
|
retryHint: "Request appears invalid for this endpoint. Fix parameters/payload before retrying.",
|
|
40
40
|
};
|
|
41
41
|
}
|
|
42
|
+
const isServerPermissionLike = status >= 500
|
|
43
|
+
&& (lowerBody.includes("not allowed to access")
|
|
44
|
+
|| lowerBody.includes("access denied")
|
|
45
|
+
|| (lowerBody.includes("permission")
|
|
46
|
+
&& (lowerBody.includes("cannot use") || lowerBody.includes("not allowed"))));
|
|
47
|
+
if (isServerPermissionLike) {
|
|
48
|
+
return {
|
|
49
|
+
category: "forbidden",
|
|
50
|
+
retryable: false,
|
|
51
|
+
retryHint: "Check API key validity and project permissions for the requested action.",
|
|
52
|
+
};
|
|
53
|
+
}
|
|
42
54
|
if (status === 404) {
|
|
43
55
|
const isHtmlGatewayResponse = lowerBody.includes("<!doctype html>");
|
|
44
56
|
return {
|
package/dist/src/index.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
export { DataikuClient, type DataikuClientConfig, } from "./client.js";
|
|
2
|
+
export { validateCredentials, } from "./auth.js";
|
|
3
|
+
export { deleteCredentials, type DssCredentials, getConfigDir, getCredentialsPath, loadCredentials, maskApiKey, saveCredentials, } from "./config.js";
|
|
2
4
|
export { DataikuError, type DataikuErrorCategory, type DataikuErrorTaxonomy, type DataikuRetryMetadata, } from "./errors.js";
|
|
3
5
|
export { CodeEnvsResource, } from "./resources/code-envs.js";
|
|
4
6
|
export { ConnectionsResource, } from "./resources/connections.js";
|
package/dist/src/index.js
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
// Client
|
|
2
2
|
export { DataikuClient, } from "./client.js";
|
|
3
|
+
// Auth & Config
|
|
4
|
+
export { validateCredentials, } from "./auth.js";
|
|
5
|
+
export { deleteCredentials, getConfigDir, getCredentialsPath, loadCredentials, maskApiKey, saveCredentials, } from "./config.js";
|
|
3
6
|
// Errors
|
|
4
7
|
export { DataikuError, } from "./errors.js";
|
|
5
8
|
// Resources (for advanced use / extension)
|
|
@@ -9,7 +9,7 @@ export declare class ConnectionsResource extends BaseResource {
|
|
|
9
9
|
* Infers available connections.
|
|
10
10
|
*
|
|
11
11
|
* - fast (default): fetches the connection name list and maps to ConnectionSummary.
|
|
12
|
-
* Falls back to rich mode on any failure.
|
|
12
|
+
* Falls back to rich mode on any failure or empty result set.
|
|
13
13
|
* - rich: inspects project datasets to derive connection metadata
|
|
14
14
|
* (types, managed flag, db schemas).
|
|
15
15
|
*/
|
|
@@ -55,7 +55,7 @@ export class ConnectionsResource extends BaseResource {
|
|
|
55
55
|
* Infers available connections.
|
|
56
56
|
*
|
|
57
57
|
* - fast (default): fetches the connection name list and maps to ConnectionSummary.
|
|
58
|
-
* Falls back to rich mode on any failure.
|
|
58
|
+
* Falls back to rich mode on any failure or empty result set.
|
|
59
59
|
* - rich: inspects project datasets to derive connection metadata
|
|
60
60
|
* (types, managed flag, db schemas).
|
|
61
61
|
*/
|
|
@@ -65,13 +65,16 @@ export class ConnectionsResource extends BaseResource {
|
|
|
65
65
|
if (mode === "rich") {
|
|
66
66
|
return inferRichConnectionsFromDatasets(this.client, projectEnc);
|
|
67
67
|
}
|
|
68
|
-
// fast — attempt name list, fall back to rich on any error
|
|
68
|
+
// fast — attempt name list, fall back to rich on any error or empty result
|
|
69
69
|
try {
|
|
70
70
|
const names = await this.list();
|
|
71
|
-
|
|
71
|
+
if (names.length > 0) {
|
|
72
|
+
return names.map((name) => ({ name, }));
|
|
73
|
+
}
|
|
72
74
|
}
|
|
73
75
|
catch {
|
|
74
|
-
|
|
76
|
+
// Fall through to rich inference.
|
|
75
77
|
}
|
|
78
|
+
return inferRichConnectionsFromDatasets(this.client, projectEnc);
|
|
76
79
|
}
|
|
77
80
|
}
|
|
@@ -346,11 +346,17 @@ export class DatasetsResource extends BaseResource {
|
|
|
346
346
|
}
|
|
347
347
|
}
|
|
348
348
|
: undefined;
|
|
349
|
+
const shouldGzip = filePath.endsWith(".gz");
|
|
349
350
|
const nodeStream = Readable.fromWeb(res.body);
|
|
350
351
|
const csvTransform = tsvToCsvTransform(downloadLimit, onHeader);
|
|
351
|
-
const gzip = createGzip();
|
|
352
352
|
const fileOut = createWriteStream(filePath);
|
|
353
|
-
|
|
353
|
+
if (shouldGzip) {
|
|
354
|
+
const gzip = createGzip();
|
|
355
|
+
await pipeline(nodeStream, csvTransform, gzip, fileOut);
|
|
356
|
+
}
|
|
357
|
+
else {
|
|
358
|
+
await pipeline(nodeStream, csvTransform, fileOut);
|
|
359
|
+
}
|
|
354
360
|
return filePath;
|
|
355
361
|
}
|
|
356
362
|
/**
|
|
@@ -2,6 +2,7 @@ import type { FolderDetails, FolderItem, FolderSummary } from "../schemas.js";
|
|
|
2
2
|
import { BaseResource } from "./base.js";
|
|
3
3
|
export declare class FoldersResource extends BaseResource {
|
|
4
4
|
list(projectKey?: string): Promise<FolderSummary[]>;
|
|
5
|
+
resolveId(nameOrId: string, projectKey?: string): Promise<string>;
|
|
5
6
|
get(folderId: string, projectKey?: string): Promise<FolderDetails>;
|
|
6
7
|
contents(folderId: string, opts?: {
|
|
7
8
|
projectKey?: string;
|
|
@@ -21,6 +21,14 @@ export class FoldersResource extends BaseResource {
|
|
|
21
21
|
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/managedfolders/`);
|
|
22
22
|
return this.client.safeParse(FolderSummaryArraySchema, raw, "folders.list");
|
|
23
23
|
}
|
|
24
|
+
async resolveId(nameOrId, projectKey) {
|
|
25
|
+
const folders = await this.list(projectKey);
|
|
26
|
+
if (folders.some((folder) => folder.id === nameOrId)) {
|
|
27
|
+
return nameOrId;
|
|
28
|
+
}
|
|
29
|
+
const match = folders.find((folder) => folder.name === nameOrId);
|
|
30
|
+
return match?.id ?? nameOrId;
|
|
31
|
+
}
|
|
24
32
|
async get(folderId, projectKey) {
|
|
25
33
|
const fEnc = encodeURIComponent(folderId);
|
|
26
34
|
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/managedfolders/${fEnc}`);
|
|
@@ -18,7 +18,8 @@ export declare class JobsResource extends BaseResource {
|
|
|
18
18
|
get(jobId: string, projectKey?: string): Promise<Record<string, unknown>>;
|
|
19
19
|
/**
|
|
20
20
|
* Retrieve job log text.
|
|
21
|
-
* Returns the last `maxLogLines` lines (default
|
|
21
|
+
* Returns the last `maxLogLines` lines (default 500) from the tail.
|
|
22
|
+
* Use `0` or `-1` to return the full log without truncation.
|
|
22
23
|
*/
|
|
23
24
|
log(jobId: string, opts?: {
|
|
24
25
|
activity?: string;
|
|
@@ -3,7 +3,7 @@ import { BaseResource, } from "./base.js";
|
|
|
3
3
|
const DEFAULT_POLL_INTERVAL_MS = 2_000;
|
|
4
4
|
const MAX_POLL_INTERVAL_MS = 10_000;
|
|
5
5
|
const DEFAULT_TIMEOUT_MS = 120_000;
|
|
6
|
-
const DEFAULT_MAX_LOG_LINES =
|
|
6
|
+
const DEFAULT_MAX_LOG_LINES = 500;
|
|
7
7
|
const TERMINAL_STATES = new Set([
|
|
8
8
|
"DONE",
|
|
9
9
|
"FAILED",
|
|
@@ -49,7 +49,8 @@ export class JobsResource extends BaseResource {
|
|
|
49
49
|
}
|
|
50
50
|
/**
|
|
51
51
|
* Retrieve job log text.
|
|
52
|
-
* Returns the last `maxLogLines` lines (default
|
|
52
|
+
* Returns the last `maxLogLines` lines (default 500) from the tail.
|
|
53
|
+
* Use `0` or `-1` to return the full log without truncation.
|
|
53
54
|
*/
|
|
54
55
|
async log(jobId, opts) {
|
|
55
56
|
const jobEnc = encodeURIComponent(jobId);
|
|
@@ -57,8 +58,11 @@ export class JobsResource extends BaseResource {
|
|
|
57
58
|
const log = await this.client.getText(`/public/api/projects/${this.enc(opts?.projectKey)}/jobs/${jobEnc}/log/${query}`);
|
|
58
59
|
if (!log)
|
|
59
60
|
return "";
|
|
60
|
-
const lines = log.split("\n");
|
|
61
61
|
const limit = opts?.maxLogLines ?? DEFAULT_MAX_LOG_LINES;
|
|
62
|
+
if (limit === 0 || limit === -1) {
|
|
63
|
+
return log;
|
|
64
|
+
}
|
|
65
|
+
const lines = log.split("\n");
|
|
62
66
|
if (lines.length > limit) {
|
|
63
67
|
return lines.slice(-limit).join("\n");
|
|
64
68
|
}
|
|
@@ -7,13 +7,28 @@ export declare class NotebooksResource extends BaseResource {
|
|
|
7
7
|
getJupyter(name: string, projectKey?: string): Promise<JupyterNotebookContent>;
|
|
8
8
|
/** Save (overwrite) a Jupyter notebook's content. */
|
|
9
9
|
saveJupyter(name: string, content: JupyterNotebookContent, projectKey?: string): Promise<void>;
|
|
10
|
-
/**
|
|
10
|
+
/**
|
|
11
|
+
* Delete a Jupyter notebook.
|
|
12
|
+
*
|
|
13
|
+
* DSS public APIs can delete notebooks but do not expose notebook creation, so
|
|
14
|
+
* this can only target notebooks created outside this SDK (for example in the UI).
|
|
15
|
+
*/
|
|
11
16
|
deleteJupyter(name: string, projectKey?: string): Promise<void>;
|
|
12
|
-
/**
|
|
17
|
+
/**
|
|
18
|
+
* Clear all cell outputs from a Jupyter notebook.
|
|
19
|
+
*
|
|
20
|
+
* DSS public APIs do not expose a dedicated clear-outputs endpoint, so this
|
|
21
|
+
* method fetches the notebook, strips outputs locally, and saves it back.
|
|
22
|
+
*/
|
|
13
23
|
clearJupyterOutputs(name: string, projectKey?: string): Promise<void>;
|
|
14
24
|
/** List running kernel sessions for a Jupyter notebook. */
|
|
15
25
|
listJupyterSessions(name: string, projectKey?: string): Promise<NotebookSession[]>;
|
|
16
|
-
/**
|
|
26
|
+
/**
|
|
27
|
+
* Unload (stop) a running Jupyter notebook session.
|
|
28
|
+
*
|
|
29
|
+
* DSS public APIs do not expose notebook or session creation, so this only
|
|
30
|
+
* works for sessions started outside this SDK.
|
|
31
|
+
*/
|
|
17
32
|
unloadJupyter(name: string, sessionId: string, projectKey?: string): Promise<void>;
|
|
18
33
|
/** List all SQL notebooks in a project. */
|
|
19
34
|
listSql(projectKey?: string): Promise<SqlNotebookSummary[]>;
|
|
@@ -18,15 +18,30 @@ export class NotebooksResource extends BaseResource {
|
|
|
18
18
|
const nameEnc = encodeURIComponent(name);
|
|
19
19
|
await this.client.putVoid(`/public/api/projects/${this.enc(projectKey)}/jupyter-notebooks/${nameEnc}`, content);
|
|
20
20
|
}
|
|
21
|
-
/**
|
|
21
|
+
/**
|
|
22
|
+
* Delete a Jupyter notebook.
|
|
23
|
+
*
|
|
24
|
+
* DSS public APIs can delete notebooks but do not expose notebook creation, so
|
|
25
|
+
* this can only target notebooks created outside this SDK (for example in the UI).
|
|
26
|
+
*/
|
|
22
27
|
async deleteJupyter(name, projectKey) {
|
|
23
28
|
const nameEnc = encodeURIComponent(name);
|
|
24
29
|
await this.client.del(`/public/api/projects/${this.enc(projectKey)}/jupyter-notebooks/${nameEnc}`);
|
|
25
30
|
}
|
|
26
|
-
/**
|
|
31
|
+
/**
|
|
32
|
+
* Clear all cell outputs from a Jupyter notebook.
|
|
33
|
+
*
|
|
34
|
+
* DSS public APIs do not expose a dedicated clear-outputs endpoint, so this
|
|
35
|
+
* method fetches the notebook, strips outputs locally, and saves it back.
|
|
36
|
+
*/
|
|
27
37
|
async clearJupyterOutputs(name, projectKey) {
|
|
28
|
-
const
|
|
29
|
-
|
|
38
|
+
const notebook = await this.getJupyter(name, projectKey);
|
|
39
|
+
const clearedCells = notebook.cells.map((cell) => ({
|
|
40
|
+
...cell,
|
|
41
|
+
outputs: [],
|
|
42
|
+
execution_count: null,
|
|
43
|
+
}));
|
|
44
|
+
await this.saveJupyter(name, { ...notebook, cells: clearedCells, }, projectKey);
|
|
30
45
|
}
|
|
31
46
|
/** List running kernel sessions for a Jupyter notebook. */
|
|
32
47
|
async listJupyterSessions(name, projectKey) {
|
|
@@ -34,7 +49,12 @@ export class NotebooksResource extends BaseResource {
|
|
|
34
49
|
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/jupyter-notebooks/${nameEnc}/sessions`);
|
|
35
50
|
return this.client.safeParse(NotebookSessionArraySchema, raw, "notebooks.sessionsJupyter");
|
|
36
51
|
}
|
|
37
|
-
/**
|
|
52
|
+
/**
|
|
53
|
+
* Unload (stop) a running Jupyter notebook session.
|
|
54
|
+
*
|
|
55
|
+
* DSS public APIs do not expose notebook or session creation, so this only
|
|
56
|
+
* works for sessions started outside this SDK.
|
|
57
|
+
*/
|
|
38
58
|
async unloadJupyter(name, sessionId, projectKey) {
|
|
39
59
|
const nameEnc = encodeURIComponent(name);
|
|
40
60
|
const sidEnc = encodeURIComponent(sessionId);
|
|
@@ -19,9 +19,26 @@ export declare class RecipesResource extends BaseResource {
|
|
|
19
19
|
create(opts: RecipeCreateOptions): Promise<RecipeCreateResult>;
|
|
20
20
|
/**
|
|
21
21
|
* Update a recipe by merging the patch into the current definition.
|
|
22
|
-
* The `recipe` sub-object is
|
|
22
|
+
* The `recipe` sub-object is deep-merged to preserve nested fields.
|
|
23
23
|
*/
|
|
24
24
|
update(recipeName: string, data: Record<string, unknown>, projectKey?: string): Promise<void>;
|
|
25
|
+
/**
|
|
26
|
+
* Download a recipe code payload to a local file.
|
|
27
|
+
|
|
28
|
+
* Returns the path to the written file.
|
|
29
|
+
*/
|
|
30
|
+
downloadCode(recipeName: string, opts?: {
|
|
31
|
+
outputPath?: string;
|
|
32
|
+
projectKey?: string;
|
|
33
|
+
}): Promise<string>;
|
|
34
|
+
/** Get only the code payload of a recipe as a raw string. */
|
|
35
|
+
getPayload(recipeName: string, opts?: {
|
|
36
|
+
projectKey?: string;
|
|
37
|
+
}): Promise<string>;
|
|
38
|
+
/** Replace only the code payload of a recipe. */
|
|
39
|
+
setPayload(recipeName: string, payload: string, opts?: {
|
|
40
|
+
projectKey?: string;
|
|
41
|
+
}): Promise<void>;
|
|
25
42
|
/** Delete a recipe. */
|
|
26
43
|
delete(recipeName: string, projectKey?: string): Promise<void>;
|
|
27
44
|
/**
|
|
@@ -2,6 +2,7 @@ import { writeFile, } from "node:fs/promises";
|
|
|
2
2
|
import { resolve, } from "node:path";
|
|
3
3
|
import { DataikuError, } from "../errors.js";
|
|
4
4
|
import { RecipeSummaryArraySchema, } from "../schemas.js";
|
|
5
|
+
import { deepMerge, } from "../utils/deep-merge.js";
|
|
5
6
|
import { sanitizeFileName, } from "../utils/sanitize.js";
|
|
6
7
|
import { BaseResource, } from "./base.js";
|
|
7
8
|
// ---------------------------------------------------------------------------
|
|
@@ -21,6 +22,22 @@ function asRecord(value) {
|
|
|
21
22
|
return undefined;
|
|
22
23
|
return value;
|
|
23
24
|
}
|
|
25
|
+
function inferRecipeCodeExtension(recipeType) {
|
|
26
|
+
const normalized = typeof recipeType === "string" ? recipeType.trim().toLowerCase() : "";
|
|
27
|
+
if (!normalized)
|
|
28
|
+
return ".txt";
|
|
29
|
+
if (normalized.includes("python") || normalized.includes("pyspark"))
|
|
30
|
+
return ".py";
|
|
31
|
+
if (normalized.includes("sql"))
|
|
32
|
+
return ".sql";
|
|
33
|
+
if (normalized === "r" || normalized.startsWith("r_"))
|
|
34
|
+
return ".R";
|
|
35
|
+
if (normalized.includes("scala"))
|
|
36
|
+
return ".scala";
|
|
37
|
+
if (normalized.includes("shell"))
|
|
38
|
+
return ".sh";
|
|
39
|
+
return ".txt";
|
|
40
|
+
}
|
|
24
41
|
// ---------------------------------------------------------------------------
|
|
25
42
|
// Helpers: retry predicate
|
|
26
43
|
// ---------------------------------------------------------------------------
|
|
@@ -53,7 +70,7 @@ export class RecipesResource extends BaseResource {
|
|
|
53
70
|
* Get a recipe definition (and optionally its payload).
|
|
54
71
|
* Returns the raw API response shape: `{ recipe, payload }`.
|
|
55
72
|
*/
|
|
56
|
-
get(recipeName, opts) {
|
|
73
|
+
async get(recipeName, opts) {
|
|
57
74
|
const enc = this.enc(opts?.projectKey);
|
|
58
75
|
const rnEnc = encodeURIComponent(recipeName);
|
|
59
76
|
const params = new URLSearchParams();
|
|
@@ -64,7 +81,12 @@ export class RecipesResource extends BaseResource {
|
|
|
64
81
|
params.set("payloadMaxLines", String(opts.payloadMaxLines));
|
|
65
82
|
const qs = params.toString();
|
|
66
83
|
const url = `/public/api/projects/${enc}/recipes/${rnEnc}${qs ? `?${qs}` : ""}`;
|
|
67
|
-
|
|
84
|
+
const result = await this.client.get(url);
|
|
85
|
+
const recipe = asRecord(result?.recipe);
|
|
86
|
+
if (!result || !recipe) {
|
|
87
|
+
throw new DataikuError(404, "Not Found", `Recipe "${recipeName}" not found in project "${this.resolveProjectKey(opts?.projectKey)}" (DSS returned empty response).`);
|
|
88
|
+
}
|
|
89
|
+
return { ...result, recipe, };
|
|
68
90
|
}
|
|
69
91
|
/** Create a recipe, with optional output dataset provisioning and join configuration. */
|
|
70
92
|
async create(opts) {
|
|
@@ -242,7 +264,7 @@ export class RecipesResource extends BaseResource {
|
|
|
242
264
|
}
|
|
243
265
|
/**
|
|
244
266
|
* Update a recipe by merging the patch into the current definition.
|
|
245
|
-
* The `recipe` sub-object is
|
|
267
|
+
* The `recipe` sub-object is deep-merged to preserve nested fields.
|
|
246
268
|
*/
|
|
247
269
|
async update(recipeName, data, projectKey) {
|
|
248
270
|
const enc = this.enc(projectKey);
|
|
@@ -252,13 +274,52 @@ export class RecipesResource extends BaseResource {
|
|
|
252
274
|
if (!currentRecipe) {
|
|
253
275
|
throw new Error(`Recipe "${recipeName}" was not found or returned an empty definition.`);
|
|
254
276
|
}
|
|
255
|
-
const mergedRecipe = {
|
|
256
|
-
...currentRecipe,
|
|
257
|
-
...data.recipe,
|
|
258
|
-
};
|
|
277
|
+
const mergedRecipe = deepMerge(currentRecipe, asRecord(data.recipe) ?? {});
|
|
259
278
|
const merged = { ...current, ...data, recipe: mergedRecipe, };
|
|
260
279
|
await this.client.put(`/public/api/projects/${enc}/recipes/${rnEnc}`, merged);
|
|
261
280
|
}
|
|
281
|
+
/**
|
|
282
|
+
* Download a recipe code payload to a local file.
|
|
283
|
+
|
|
284
|
+
* Returns the path to the written file.
|
|
285
|
+
*/
|
|
286
|
+
async downloadCode(recipeName, opts) {
|
|
287
|
+
const result = await this.get(recipeName, {
|
|
288
|
+
includePayload: true,
|
|
289
|
+
projectKey: opts?.projectKey,
|
|
290
|
+
});
|
|
291
|
+
if (!result.payload) {
|
|
292
|
+
throw new Error(`Recipe "${recipeName}" has no code payload.`);
|
|
293
|
+
}
|
|
294
|
+
const safeRecipeName = sanitizeFileName(recipeName, "recipe");
|
|
295
|
+
const filePath = opts?.outputPath ?? resolve(process.cwd(), `${safeRecipeName}${inferRecipeCodeExtension(result.recipe.type)}`);
|
|
296
|
+
await writeFile(filePath, result.payload, "utf-8");
|
|
297
|
+
return filePath;
|
|
298
|
+
}
|
|
299
|
+
/** Get only the code payload of a recipe as a raw string. */
|
|
300
|
+
async getPayload(recipeName, opts) {
|
|
301
|
+
const result = await this.get(recipeName, {
|
|
302
|
+
includePayload: true,
|
|
303
|
+
projectKey: opts?.projectKey,
|
|
304
|
+
});
|
|
305
|
+
if (!result.payload) {
|
|
306
|
+
throw new Error(`Recipe "${recipeName}" has no code payload.`);
|
|
307
|
+
}
|
|
308
|
+
return result.payload;
|
|
309
|
+
}
|
|
310
|
+
/** Replace only the code payload of a recipe. */
|
|
311
|
+
async setPayload(recipeName, payload, opts) {
|
|
312
|
+
const current = await this.get(recipeName, {
|
|
313
|
+
includePayload: true,
|
|
314
|
+
projectKey: opts?.projectKey,
|
|
315
|
+
});
|
|
316
|
+
const enc = this.enc(opts?.projectKey);
|
|
317
|
+
const rnEnc = encodeURIComponent(recipeName);
|
|
318
|
+
await this.client.put(`/public/api/projects/${enc}/recipes/${rnEnc}`, {
|
|
319
|
+
...current,
|
|
320
|
+
payload,
|
|
321
|
+
});
|
|
322
|
+
}
|
|
262
323
|
/** Delete a recipe. */
|
|
263
324
|
async delete(recipeName, projectKey) {
|
|
264
325
|
const enc = this.enc(projectKey);
|
|
@@ -1,4 +1,18 @@
|
|
|
1
|
+
import { DataikuError, } from "../errors.js";
|
|
1
2
|
import { BaseResource, } from "./base.js";
|
|
3
|
+
const UNSUPPORTED_SQL_DATASET_CONNECTION_DETAIL = "neither of sql nor hdfs type";
|
|
4
|
+
function isUnsupportedSqlDatasetConnectionError(error) {
|
|
5
|
+
if (!(error instanceof DataikuError))
|
|
6
|
+
return false;
|
|
7
|
+
const detail = `${error.statusText}\n${error.body}\n${error.message}`.toLowerCase();
|
|
8
|
+
return detail.includes(UNSUPPORTED_SQL_DATASET_CONNECTION_DETAIL);
|
|
9
|
+
}
|
|
10
|
+
function buildUnsupportedSqlDatasetConnectionMessage(datasetFullName) {
|
|
11
|
+
const subject = datasetFullName
|
|
12
|
+
? `Dataset "${datasetFullName}" uses a connection that DSS does not support for direct SQL queries.`
|
|
13
|
+
: "This query uses a connection that DSS does not support for direct SQL queries.";
|
|
14
|
+
return `${subject} Use --connection with a SQL-compatible connection instead.`;
|
|
15
|
+
}
|
|
2
16
|
export class SqlResource extends BaseResource {
|
|
3
17
|
/**
|
|
4
18
|
* Start a SQL query and return the queryId + schema.
|
|
@@ -6,7 +20,10 @@ export class SqlResource extends BaseResource {
|
|
|
6
20
|
* or `datasetFullName` (run against a dataset's connection).
|
|
7
21
|
*/
|
|
8
22
|
async startQuery(opts) {
|
|
9
|
-
return this.client.post(
|
|
23
|
+
return this.client.post("/public/api/sql/queries/", {
|
|
24
|
+
...opts,
|
|
25
|
+
type: opts.type ?? "sql",
|
|
26
|
+
});
|
|
10
27
|
}
|
|
11
28
|
/**
|
|
12
29
|
* Stream results of a started query as parsed JSON (array of arrays).
|
|
@@ -32,9 +49,19 @@ export class SqlResource extends BaseResource {
|
|
|
32
49
|
* This is the primary method most callers want.
|
|
33
50
|
*/
|
|
34
51
|
async query(opts) {
|
|
35
|
-
const {
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
52
|
+
const queryOpts = { ...opts, type: opts.type ?? "sql", };
|
|
53
|
+
try {
|
|
54
|
+
const { queryId, schema, } = await this.startQuery(queryOpts);
|
|
55
|
+
const rows = await this.streamResults(queryId);
|
|
56
|
+
await this.finishStreaming(queryId);
|
|
57
|
+
return { queryId, schema, rows, };
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
if (!isUnsupportedSqlDatasetConnectionError(error))
|
|
61
|
+
throw error;
|
|
62
|
+
throw new Error(buildUnsupportedSqlDatasetConnectionMessage(queryOpts.datasetFullName), {
|
|
63
|
+
cause: error,
|
|
64
|
+
});
|
|
65
|
+
}
|
|
39
66
|
}
|
|
40
67
|
}
|
|
@@ -7,6 +7,15 @@ export class VariablesResource extends BaseResource {
|
|
|
7
7
|
return this.client.safeParse(ProjectVariablesSchema, raw, "variables.get");
|
|
8
8
|
}
|
|
9
9
|
async set(opts) {
|
|
10
|
+
const enc = this.enc(opts.projectKey);
|
|
11
|
+
if (opts.replace === true) {
|
|
12
|
+
const replaced = {
|
|
13
|
+
standard: opts.standard ?? {},
|
|
14
|
+
local: opts.local ?? {},
|
|
15
|
+
};
|
|
16
|
+
await this.client.putVoid(`/public/api/projects/${enc}/variables/`, replaced);
|
|
17
|
+
return replaced;
|
|
18
|
+
}
|
|
10
19
|
if (opts.standard === undefined && opts.local === undefined) {
|
|
11
20
|
throw new Error("At least one of standard or local must be provided");
|
|
12
21
|
}
|
|
@@ -15,7 +24,6 @@ export class VariablesResource extends BaseResource {
|
|
|
15
24
|
standard: { ...existing.standard, ...opts.standard, },
|
|
16
25
|
local: { ...existing.local, ...opts.local, },
|
|
17
26
|
};
|
|
18
|
-
const enc = this.enc(opts.projectKey);
|
|
19
27
|
await this.client.putVoid(`/public/api/projects/${enc}/variables/`, merged);
|
|
20
28
|
return merged;
|
|
21
29
|
}
|