dataiku-sdk 0.2.4 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/auth.d.ts +5 -1
- package/dist/src/auth.js +3 -1
- package/dist/src/cli.js +106 -27
- package/dist/src/client.d.ts +6 -0
- package/dist/src/client.js +36 -1
- package/dist/src/config.d.ts +2 -0
- package/dist/src/config.js +9 -0
- package/dist/src/index.d.ts +1 -1
- package/dist/src/resources/datasets.d.ts +1 -0
- package/dist/src/resources/datasets.js +56 -25
- package/dist/src/resources/sql.d.ts +16 -18
- package/dist/src/resources/sql.js +72 -7
- package/dist/src/skill.js +3 -1
- package/package.json +1 -1
package/dist/src/auth.d.ts
CHANGED
|
@@ -4,4 +4,8 @@ export interface CredentialValidationResult {
|
|
|
4
4
|
error?: string;
|
|
5
5
|
dataikuError?: DataikuError;
|
|
6
6
|
}
|
|
7
|
-
export
|
|
7
|
+
export interface CredentialValidationOptions {
|
|
8
|
+
tlsRejectUnauthorized?: boolean;
|
|
9
|
+
caCertPath?: string;
|
|
10
|
+
}
|
|
11
|
+
export declare function validateCredentials(url: string, apiKey: string, options?: CredentialValidationOptions): Promise<CredentialValidationResult>;
|
package/dist/src/auth.js
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { DataikuClient, } from "./client.js";
|
|
2
2
|
import { DataikuError, } from "./errors.js";
|
|
3
|
-
export async function validateCredentials(url, apiKey) {
|
|
3
|
+
export async function validateCredentials(url, apiKey, options = {}) {
|
|
4
4
|
try {
|
|
5
5
|
const client = new DataikuClient({
|
|
6
6
|
url,
|
|
7
7
|
apiKey,
|
|
8
8
|
requestTimeoutMs: 10_000,
|
|
9
9
|
retryMaxAttempts: 1,
|
|
10
|
+
tlsRejectUnauthorized: options.tlsRejectUnauthorized,
|
|
11
|
+
caCertPath: options.caCertPath,
|
|
10
12
|
});
|
|
11
13
|
await client.projects.list();
|
|
12
14
|
return { valid: true, };
|
package/dist/src/cli.js
CHANGED
|
@@ -42,9 +42,13 @@ function json(v) {
|
|
|
42
42
|
return undefined;
|
|
43
43
|
return JSON.parse(v);
|
|
44
44
|
}
|
|
45
|
+
const SQL_QUERY_USAGE = "dss sql query [SQL | --sql QUERY | --sql-file PATH | --sql - | --stdin] (--connection CONN | --dataset FULL_NAME) [--database DB] [--project-key KEY]";
|
|
46
|
+
function readStdinText() {
|
|
47
|
+
return readFileSync(0, "utf-8");
|
|
48
|
+
}
|
|
45
49
|
function jsonInput(flags) {
|
|
46
50
|
if (flags["stdin"] === true) {
|
|
47
|
-
return JSON.parse(
|
|
51
|
+
return JSON.parse(readStdinText());
|
|
48
52
|
}
|
|
49
53
|
if (typeof flags["data-file"] === "string") {
|
|
50
54
|
return JSON.parse(readFileSync(flags["data-file"], "utf-8"));
|
|
@@ -54,6 +58,62 @@ function jsonInput(flags) {
|
|
|
54
58
|
}
|
|
55
59
|
return undefined;
|
|
56
60
|
}
|
|
61
|
+
function parseTlsRejectUnauthorizedEnv(value) {
|
|
62
|
+
if (value === undefined)
|
|
63
|
+
return undefined;
|
|
64
|
+
const normalized = value.trim().toLowerCase();
|
|
65
|
+
if (normalized === "0" || normalized === "false" || normalized === "no")
|
|
66
|
+
return false;
|
|
67
|
+
if (normalized === "1" || normalized === "true" || normalized === "yes")
|
|
68
|
+
return true;
|
|
69
|
+
return undefined;
|
|
70
|
+
}
|
|
71
|
+
function resolveTlsSettings(flags, saved) {
|
|
72
|
+
let tlsRejectUnauthorized = flags["insecure"] === true ? false : undefined;
|
|
73
|
+
let caCertPath = flags["ca-cert"];
|
|
74
|
+
tlsRejectUnauthorized ??= parseTlsRejectUnauthorizedEnv(process.env.NODE_TLS_REJECT_UNAUTHORIZED);
|
|
75
|
+
caCertPath ??= process.env.NODE_EXTRA_CA_CERTS;
|
|
76
|
+
if (tlsRejectUnauthorized === undefined) {
|
|
77
|
+
tlsRejectUnauthorized = saved?.tlsRejectUnauthorized;
|
|
78
|
+
}
|
|
79
|
+
caCertPath ??= saved?.caCertPath;
|
|
80
|
+
return { tlsRejectUnauthorized, caCertPath, };
|
|
81
|
+
}
|
|
82
|
+
function resolveSqlInput(args, flags) {
|
|
83
|
+
const sources = [];
|
|
84
|
+
if (typeof flags["sql"] === "string") {
|
|
85
|
+
sources.push({
|
|
86
|
+
label: flags["sql"] === "-" ? "--sql -" : "--sql",
|
|
87
|
+
read: () => flags["sql"] === "-" ? readStdinText() : String(flags["sql"]),
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
if (typeof flags["sql-file"] === "string") {
|
|
91
|
+
sources.push({
|
|
92
|
+
label: "--sql-file",
|
|
93
|
+
read: () => readFileSync(flags["sql-file"], "utf-8"),
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
if (flags["stdin"] === true) {
|
|
97
|
+
sources.push({ label: "--stdin", read: readStdinText, });
|
|
98
|
+
}
|
|
99
|
+
if (args.length > 1) {
|
|
100
|
+
throw new UsageError(`Expected at most one positional SQL argument. Quote the SQL or use --sql-file/--stdin.\nUsage: ${SQL_QUERY_USAGE}`);
|
|
101
|
+
}
|
|
102
|
+
if (args[0] !== undefined) {
|
|
103
|
+
sources.push({ label: "positional SQL", read: () => args[0], });
|
|
104
|
+
}
|
|
105
|
+
if (sources.length === 0) {
|
|
106
|
+
throw new UsageError(`SQL input is required. Usage: ${SQL_QUERY_USAGE}`);
|
|
107
|
+
}
|
|
108
|
+
if (sources.length > 1) {
|
|
109
|
+
throw new UsageError(`Choose exactly one SQL input source: --sql, --sql-file, --stdin, or one positional SQL argument. Usage: ${SQL_QUERY_USAGE}`);
|
|
110
|
+
}
|
|
111
|
+
const query = sources[0].read();
|
|
112
|
+
if (query.trim().length === 0) {
|
|
113
|
+
throw new UsageError(`SQL input from ${sources[0].label} must not be empty. Usage: ${SQL_QUERY_USAGE}`);
|
|
114
|
+
}
|
|
115
|
+
return query;
|
|
116
|
+
}
|
|
57
117
|
async function resolveFolderId(client, nameOrId, flags) {
|
|
58
118
|
return client.folders.resolveId(nameOrId, flags["project-key"]);
|
|
59
119
|
}
|
|
@@ -155,6 +215,7 @@ const BOOLEAN_FLAGS = new Set([
|
|
|
155
215
|
"verbose",
|
|
156
216
|
"version",
|
|
157
217
|
"stdin",
|
|
218
|
+
"insecure",
|
|
158
219
|
"global",
|
|
159
220
|
"list-agents",
|
|
160
221
|
"include-raw",
|
|
@@ -172,6 +233,8 @@ const SHORT_FLAGS = {
|
|
|
172
233
|
/** Long-flag aliases: these are normalized to the canonical name in parseArgs. */
|
|
173
234
|
const FLAG_ALIASES = {
|
|
174
235
|
project: "project-key",
|
|
236
|
+
"skip-tls-verify": "insecure",
|
|
237
|
+
"extra-ca-certs": "ca-cert",
|
|
175
238
|
};
|
|
176
239
|
function isNegativeNumberToken(value) {
|
|
177
240
|
return value.startsWith("-") && Number.isFinite(Number(value));
|
|
@@ -286,9 +349,10 @@ const commands = {
|
|
|
286
349
|
return c.datasets.preview(a[0], {
|
|
287
350
|
maxRows: num(f["max-rows"]),
|
|
288
351
|
projectKey: f["project-key"],
|
|
352
|
+
timeoutMs: num(f["timeout"]),
|
|
289
353
|
});
|
|
290
354
|
},
|
|
291
|
-
usage: "dss dataset preview <name> [--max-rows N] [--project-key KEY]",
|
|
355
|
+
usage: "dss dataset preview <name> [--max-rows N] [--project-key KEY] [--timeout MS]",
|
|
292
356
|
},
|
|
293
357
|
metadata: {
|
|
294
358
|
handler: (c, a, f) => {
|
|
@@ -661,18 +725,22 @@ const commands = {
|
|
|
661
725
|
},
|
|
662
726
|
sql: {
|
|
663
727
|
query: {
|
|
664
|
-
handler: (c,
|
|
665
|
-
const query = f
|
|
666
|
-
|
|
667
|
-
|
|
728
|
+
handler: (c, a, f) => {
|
|
729
|
+
const query = resolveSqlInput(a, f);
|
|
730
|
+
const connection = f["connection"];
|
|
731
|
+
const datasetFullName = f["dataset"];
|
|
732
|
+
if ((connection ? 1 : 0) + (datasetFullName ? 1 : 0) !== 1) {
|
|
733
|
+
throw new UsageError(`Pass exactly one of --connection or --dataset. Usage: ${SQL_QUERY_USAGE}`);
|
|
734
|
+
}
|
|
668
735
|
return c.sql.query({
|
|
669
736
|
query,
|
|
670
|
-
connection
|
|
671
|
-
datasetFullName
|
|
737
|
+
connection,
|
|
738
|
+
datasetFullName,
|
|
672
739
|
database: f["database"],
|
|
740
|
+
projectKey: f["project-key"],
|
|
673
741
|
});
|
|
674
742
|
},
|
|
675
|
-
usage:
|
|
743
|
+
usage: SQL_QUERY_USAGE,
|
|
676
744
|
},
|
|
677
745
|
},
|
|
678
746
|
notebook: {
|
|
@@ -793,6 +861,8 @@ function printTopLevelHelp() {
|
|
|
793
861
|
" --api-key KEY API key (env: DATAIKU_API_KEY)",
|
|
794
862
|
" --project-key KEY Default project key (env: DATAIKU_PROJECT_KEY)",
|
|
795
863
|
" --timeout MS Request timeout in ms (default: 30000)",
|
|
864
|
+
" --insecure Disable TLS certificate verification",
|
|
865
|
+
" --ca-cert PATH Extra PEM CA bundle (env: NODE_EXTRA_CA_CERTS)",
|
|
796
866
|
"",
|
|
797
867
|
"Resources:",
|
|
798
868
|
...RESOURCE_NAMES.map((r) => ` ${r}`),
|
|
@@ -877,6 +947,7 @@ function loadEnvFile() {
|
|
|
877
947
|
const AUTH_ACTIONS = {
|
|
878
948
|
login: {
|
|
879
949
|
handler: async (flags) => {
|
|
950
|
+
const tlsSettings = resolveTlsSettings(flags);
|
|
880
951
|
let { url, apiKey, projectKey, } = resolveCredentials(flags);
|
|
881
952
|
if (!url || !apiKey) {
|
|
882
953
|
if (!process.stdin.isTTY) {
|
|
@@ -894,43 +965,46 @@ const AUTH_ACTIONS = {
|
|
|
894
965
|
if (!apiKey)
|
|
895
966
|
throw new UsageError("API key is required.");
|
|
896
967
|
process.stderr.write("Validating credentials... ");
|
|
897
|
-
const result = await validateCredentials(url, apiKey);
|
|
968
|
+
const result = await validateCredentials(url, apiKey, tlsSettings);
|
|
898
969
|
if (!result.valid) {
|
|
899
970
|
process.stderr.write(`✗ Failed\n`);
|
|
900
971
|
if (result.dataikuError)
|
|
901
972
|
throw result.dataikuError;
|
|
902
973
|
throw new DataikuError(0, "Authentication Failed", result.error ?? "Credential validation failed");
|
|
903
974
|
}
|
|
904
|
-
process.stderr.write("
|
|
905
|
-
saveCredentials({ url, apiKey, projectKey, });
|
|
975
|
+
process.stderr.write("✓ Connected\n");
|
|
976
|
+
saveCredentials({ url, apiKey, projectKey, ...tlsSettings, });
|
|
906
977
|
process.stderr.write(`Credentials saved to ${getCredentialsPath()}\n`);
|
|
907
978
|
},
|
|
908
|
-
usage: "dss auth login [--url URL] [--api-key KEY] [--project-key KEY]",
|
|
979
|
+
usage: "dss auth login [--url URL] [--api-key KEY] [--project-key KEY] [--insecure] [--ca-cert PATH]",
|
|
909
980
|
},
|
|
910
981
|
status: {
|
|
911
|
-
handler: async (
|
|
982
|
+
handler: async (flags) => {
|
|
912
983
|
const creds = loadCredentials();
|
|
913
984
|
if (!creds) {
|
|
914
985
|
process.stderr.write("No saved credentials. Run: dss auth login\n");
|
|
915
986
|
return;
|
|
916
987
|
}
|
|
988
|
+
const tlsSettings = resolveTlsSettings(flags, creds);
|
|
917
989
|
const lines = [
|
|
918
990
|
`URL: ${creds.url}`,
|
|
919
991
|
`API key: ${maskApiKey(creds.apiKey)}`,
|
|
920
992
|
`Project key: ${creds.projectKey ?? "(not set)"}`,
|
|
993
|
+
`TLS verify: ${tlsSettings.tlsRejectUnauthorized === false ? "disabled" : "strict"}`,
|
|
994
|
+
`CA cert: ${tlsSettings.caCertPath ?? "(default trust store)"}`,
|
|
921
995
|
];
|
|
922
996
|
for (const line of lines)
|
|
923
997
|
process.stderr.write(`${line}\n`);
|
|
924
|
-
const result = await validateCredentials(creds.url, creds.apiKey);
|
|
998
|
+
const result = await validateCredentials(creds.url, creds.apiKey, tlsSettings);
|
|
925
999
|
if (result.valid) {
|
|
926
|
-
process.stderr.write("Connection:
|
|
1000
|
+
process.stderr.write("Connection: ✓ Valid\n");
|
|
927
1001
|
}
|
|
928
1002
|
else {
|
|
929
|
-
process.stderr.write(`Connection:
|
|
1003
|
+
process.stderr.write(`Connection: ✗ Failed (${result.error ?? "unknown error"})\n`);
|
|
930
1004
|
}
|
|
931
1005
|
process.stderr.write(`Config: ${getCredentialsPath()}\n`);
|
|
932
1006
|
},
|
|
933
|
-
usage: "dss auth status",
|
|
1007
|
+
usage: "dss auth status [--insecure] [--ca-cert PATH]",
|
|
934
1008
|
},
|
|
935
1009
|
logout: {
|
|
936
1010
|
handler: async (_flags) => {
|
|
@@ -977,18 +1051,21 @@ function resolveCredentials(flags) {
|
|
|
977
1051
|
let url = flags["url"];
|
|
978
1052
|
let apiKey = flags["api-key"];
|
|
979
1053
|
let projectKey = flags["project-key"];
|
|
1054
|
+
const saved = loadCredentials();
|
|
980
1055
|
url ??= process.env.DATAIKU_URL;
|
|
981
1056
|
apiKey ??= process.env.DATAIKU_API_KEY;
|
|
982
1057
|
projectKey ??= process.env.DATAIKU_PROJECT_KEY;
|
|
983
|
-
if (
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
apiKey ||= saved.apiKey;
|
|
988
|
-
projectKey ??= saved.projectKey;
|
|
989
|
-
}
|
|
1058
|
+
if (saved) {
|
|
1059
|
+
url ||= saved.url;
|
|
1060
|
+
apiKey ||= saved.apiKey;
|
|
1061
|
+
projectKey ??= saved.projectKey;
|
|
990
1062
|
}
|
|
991
|
-
return {
|
|
1063
|
+
return {
|
|
1064
|
+
url: url ?? "",
|
|
1065
|
+
apiKey: apiKey ?? "",
|
|
1066
|
+
projectKey,
|
|
1067
|
+
...resolveTlsSettings(flags, saved ?? undefined),
|
|
1068
|
+
};
|
|
992
1069
|
}
|
|
993
1070
|
// ---------------------------------------------------------------------------
|
|
994
1071
|
// Main
|
|
@@ -1121,7 +1198,7 @@ async function main() {
|
|
|
1121
1198
|
process.exit(0);
|
|
1122
1199
|
}
|
|
1123
1200
|
// Resolve credentials: flags > env > saved > .env
|
|
1124
|
-
const { url, apiKey, projectKey, } = resolveCredentials(flags);
|
|
1201
|
+
const { url, apiKey, projectKey, tlsRejectUnauthorized, caCertPath, } = resolveCredentials(flags);
|
|
1125
1202
|
if (!url) {
|
|
1126
1203
|
throw new UsageError("Missing Dataiku URL. Set DATAIKU_URL, pass --url, or run: dss auth login");
|
|
1127
1204
|
}
|
|
@@ -1135,6 +1212,8 @@ async function main() {
|
|
|
1135
1212
|
projectKey,
|
|
1136
1213
|
verbose: flags["verbose"] === true,
|
|
1137
1214
|
requestTimeoutMs,
|
|
1215
|
+
tlsRejectUnauthorized,
|
|
1216
|
+
caCertPath,
|
|
1138
1217
|
});
|
|
1139
1218
|
const args = positional.slice(2);
|
|
1140
1219
|
const format = parseOutputFormat(flags["format"]);
|
package/dist/src/client.d.ts
CHANGED
|
@@ -23,6 +23,10 @@ export interface DataikuClientConfig {
|
|
|
23
23
|
retryMaxAttempts?: number;
|
|
24
24
|
/** Emit HTTP request/response logs to stderr for CLI debugging. */
|
|
25
25
|
verbose?: boolean;
|
|
26
|
+
/** Override TLS certificate verification for HTTPS requests. */
|
|
27
|
+
tlsRejectUnauthorized?: boolean;
|
|
28
|
+
/** Extra PEM CA bundle to trust in addition to Bun's default trust store. */
|
|
29
|
+
caCertPath?: string;
|
|
26
30
|
/**
|
|
27
31
|
* Called when an API response fails schema validation but data is still usable.
|
|
28
32
|
* Default: writes to stderr. Set to a throwing function for strict mode.
|
|
@@ -38,6 +42,7 @@ export declare class DataikuClient {
|
|
|
38
42
|
private readonly requestTimeoutMs;
|
|
39
43
|
private readonly retryMaxAttempts;
|
|
40
44
|
private readonly verbose;
|
|
45
|
+
private readonly tlsOptions;
|
|
41
46
|
private readonly onValidationWarning;
|
|
42
47
|
private _projects?;
|
|
43
48
|
private _datasets?;
|
|
@@ -62,6 +67,7 @@ export declare class DataikuClient {
|
|
|
62
67
|
get sql(): SqlResource;
|
|
63
68
|
get notebooks(): NotebooksResource;
|
|
64
69
|
constructor(config: DataikuClientConfig);
|
|
70
|
+
getRequestTimeoutMs(): number;
|
|
65
71
|
resolveProjectKey(paramValue?: string): string;
|
|
66
72
|
get<T = unknown>(path: string): Promise<T>;
|
|
67
73
|
getText(path: string): Promise<string>;
|
package/dist/src/client.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { readFileSync, } from "node:fs";
|
|
2
|
+
import { getCACertificates, } from "node:tls";
|
|
1
3
|
import { Value, } from "@sinclair/typebox/value";
|
|
2
4
|
import { safeParseSchema, } from "./schemas.js";
|
|
3
5
|
import { classifyDataikuError, DataikuError, } from "./errors.js";
|
|
@@ -50,6 +52,27 @@ function buildRetryMetadata(method, enabled, maxAttempts, attempts, delaysMs, ti
|
|
|
50
52
|
timedOut,
|
|
51
53
|
};
|
|
52
54
|
}
|
|
55
|
+
function buildFetchTlsOptions(config) {
|
|
56
|
+
const rejectUnauthorized = config.tlsRejectUnauthorized;
|
|
57
|
+
const caCertPath = config.caCertPath?.trim();
|
|
58
|
+
if (rejectUnauthorized === undefined && !caCertPath)
|
|
59
|
+
return undefined;
|
|
60
|
+
const tls = {};
|
|
61
|
+
if (rejectUnauthorized !== undefined)
|
|
62
|
+
tls.rejectUnauthorized = rejectUnauthorized;
|
|
63
|
+
if (caCertPath) {
|
|
64
|
+
try {
|
|
65
|
+
tls.ca = [...getCACertificates("default"), readFileSync(caCertPath, "utf-8"),];
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
69
|
+
throw new Error(`Unable to read CA certificate bundle at ${caCertPath}: ${message}`, {
|
|
70
|
+
cause: error,
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
return tls;
|
|
75
|
+
}
|
|
53
76
|
/* ------------------------------------------------------------------ */
|
|
54
77
|
/* Client */
|
|
55
78
|
/* ------------------------------------------------------------------ */
|
|
@@ -60,6 +83,7 @@ export class DataikuClient {
|
|
|
60
83
|
requestTimeoutMs;
|
|
61
84
|
retryMaxAttempts;
|
|
62
85
|
verbose;
|
|
86
|
+
tlsOptions;
|
|
63
87
|
onValidationWarning;
|
|
64
88
|
/* Resource namespaces — lazily initialized to break circular imports */
|
|
65
89
|
_projects;
|
|
@@ -120,8 +144,12 @@ export class DataikuClient {
|
|
|
120
144
|
const rawMax = config.retryMaxAttempts ?? DEFAULT_RETRY_MAX_ATTEMPTS;
|
|
121
145
|
this.retryMaxAttempts = Math.min(Math.max(1, rawMax), MAX_RETRY_ATTEMPTS_CAP);
|
|
122
146
|
this.verbose = config.verbose === true;
|
|
147
|
+
this.tlsOptions = buildFetchTlsOptions(config);
|
|
123
148
|
this.onValidationWarning = config.onValidationWarning ?? defaultValidationWarning;
|
|
124
149
|
}
|
|
150
|
+
getRequestTimeoutMs() {
|
|
151
|
+
return this.requestTimeoutMs;
|
|
152
|
+
}
|
|
125
153
|
/* ---- public: project key resolution ---- */
|
|
126
154
|
resolveProjectKey(paramValue) {
|
|
127
155
|
const pk = paramValue?.trim();
|
|
@@ -271,7 +299,14 @@ export class DataikuClient {
|
|
|
271
299
|
}, this.requestTimeoutMs);
|
|
272
300
|
this.logVerbose(`${method} ${url}`);
|
|
273
301
|
try {
|
|
274
|
-
const
|
|
302
|
+
const requestInit = {
|
|
303
|
+
...init,
|
|
304
|
+
method,
|
|
305
|
+
signal: controller.signal,
|
|
306
|
+
};
|
|
307
|
+
if (this.tlsOptions)
|
|
308
|
+
requestInit.tls = this.tlsOptions;
|
|
309
|
+
const res = await fetch(url, requestInit);
|
|
275
310
|
this.logVerbose(`${method} ${url} → ${res.status} (${Date.now() - startedAt}ms)`);
|
|
276
311
|
if (!res.ok) {
|
|
277
312
|
const text = await res.text();
|
package/dist/src/config.d.ts
CHANGED
package/dist/src/config.js
CHANGED
|
@@ -30,6 +30,10 @@ export function loadCredentials() {
|
|
|
30
30
|
url: obj.url,
|
|
31
31
|
apiKey: obj.apiKey,
|
|
32
32
|
projectKey: typeof obj.projectKey === "string" ? obj.projectKey : undefined,
|
|
33
|
+
tlsRejectUnauthorized: typeof obj.tlsRejectUnauthorized === "boolean"
|
|
34
|
+
? obj.tlsRejectUnauthorized
|
|
35
|
+
: undefined,
|
|
36
|
+
caCertPath: typeof obj.caCertPath === "string" ? obj.caCertPath : undefined,
|
|
33
37
|
};
|
|
34
38
|
}
|
|
35
39
|
catch (err) {
|
|
@@ -44,6 +48,11 @@ export function saveCredentials(creds) {
|
|
|
44
48
|
const data = { url: creds.url, apiKey: creds.apiKey, };
|
|
45
49
|
if (creds.projectKey)
|
|
46
50
|
data.projectKey = creds.projectKey;
|
|
51
|
+
if (creds.tlsRejectUnauthorized !== undefined) {
|
|
52
|
+
data.tlsRejectUnauthorized = creds.tlsRejectUnauthorized;
|
|
53
|
+
}
|
|
54
|
+
if (creds.caCertPath)
|
|
55
|
+
data.caCertPath = creds.caCertPath;
|
|
47
56
|
writeFileSync(path, `${JSON.stringify(data, null, 2)}\n`, "utf-8");
|
|
48
57
|
chmodSync(path, 0o600);
|
|
49
58
|
}
|
package/dist/src/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export { DataikuClient, type DataikuClientConfig, } from "./client.js";
|
|
2
|
-
export { validateCredentials, } from "./auth.js";
|
|
2
|
+
export { type CredentialValidationOptions, type CredentialValidationResult, validateCredentials, } from "./auth.js";
|
|
3
3
|
export { deleteCredentials, type DssCredentials, getConfigDir, getCredentialsPath, loadCredentials, maskApiKey, saveCredentials, } from "./config.js";
|
|
4
4
|
export { DataikuError, type DataikuErrorCategory, type DataikuErrorTaxonomy, type DataikuRetryMetadata, } from "./errors.js";
|
|
5
5
|
export { CodeEnvsResource, } from "./resources/code-envs.js";
|
|
@@ -28,6 +28,7 @@ export declare class DatasetsResource extends BaseResource {
|
|
|
28
28
|
validateColumns?: {
|
|
29
29
|
name: string;
|
|
30
30
|
}[];
|
|
31
|
+
timeoutMs?: number;
|
|
31
32
|
}): Promise<string>;
|
|
32
33
|
/** Get dataset metadata (tags, custom fields, checklists). */
|
|
33
34
|
metadata(datasetName: string, projectKey?: string): Promise<Record<string, unknown>>;
|
|
@@ -149,37 +149,65 @@ function emitCsvLineWithLimit(row, maxDataRows, emittedRows, onLine, onHeader) {
|
|
|
149
149
|
}
|
|
150
150
|
return false;
|
|
151
151
|
}
|
|
152
|
-
|
|
152
|
+
function buildPreviewTimeoutError(timeoutMs) {
|
|
153
|
+
return new DataikuError(0, "Request Timeout", `Dataset preview timed out after ${timeoutMs}ms while waiting for rows.`);
|
|
154
|
+
}
|
|
155
|
+
async function readChunkWithTimeout(reader, remainingMs, timeoutMs) {
|
|
156
|
+
return new Promise((resolveChunk, rejectChunk) => {
|
|
157
|
+
const timer = setTimeout(() => {
|
|
158
|
+
void reader.cancel(buildPreviewTimeoutError(timeoutMs)).catch(() => { });
|
|
159
|
+
rejectChunk(buildPreviewTimeoutError(timeoutMs));
|
|
160
|
+
}, remainingMs);
|
|
161
|
+
reader.read().then((result) => {
|
|
162
|
+
clearTimeout(timer);
|
|
163
|
+
resolveChunk(result);
|
|
164
|
+
}, (error) => {
|
|
165
|
+
clearTimeout(timer);
|
|
166
|
+
rejectChunk(error);
|
|
167
|
+
});
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
async function collectPreviewCsv(body, maxDataRows, timeoutMs, onHeader) {
|
|
153
171
|
const state = createTsvStreamState();
|
|
154
172
|
const emittedRows = { value: 0, };
|
|
155
173
|
const lines = [];
|
|
156
174
|
let done = false;
|
|
157
|
-
const
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
175
|
+
const startedAt = Date.now();
|
|
176
|
+
const reader = body.getReader();
|
|
177
|
+
try {
|
|
178
|
+
while (true) {
|
|
179
|
+
if (done) {
|
|
180
|
+
void reader.cancel().catch(() => { });
|
|
181
|
+
break;
|
|
182
|
+
}
|
|
183
|
+
const remainingMs = timeoutMs - (Date.now() - startedAt);
|
|
184
|
+
if (remainingMs <= 0)
|
|
185
|
+
throw buildPreviewTimeoutError(timeoutMs);
|
|
186
|
+
const result = await readChunkWithTimeout(reader, remainingMs, timeoutMs);
|
|
187
|
+
if (result.done)
|
|
188
|
+
break;
|
|
189
|
+
consumeTsvChunk(Buffer.from(result.value).toString("utf-8"), state, (row) => {
|
|
190
|
+
if (done)
|
|
191
|
+
return;
|
|
192
|
+
done = emitCsvLineWithLimit(row, maxDataRows, emittedRows, (line) => {
|
|
193
|
+
lines.push(line);
|
|
194
|
+
}, onHeader);
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
if (!done) {
|
|
198
|
+
flushTsvStream(state, (row) => {
|
|
199
|
+
if (done)
|
|
200
|
+
return;
|
|
201
|
+
done = emitCsvLineWithLimit(row, maxDataRows, emittedRows, (line) => {
|
|
202
|
+
lines.push(line);
|
|
203
|
+
}, onHeader);
|
|
204
|
+
});
|
|
171
205
|
}
|
|
206
|
+
return lines.join("\n");
|
|
172
207
|
}
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
if (done)
|
|
176
|
-
return;
|
|
177
|
-
done = emitCsvLineWithLimit(row, maxDataRows, emittedRows, (line) => {
|
|
178
|
-
lines.push(line);
|
|
179
|
-
}, onHeader);
|
|
180
|
-
});
|
|
208
|
+
finally {
|
|
209
|
+
reader.releaseLock();
|
|
181
210
|
}
|
|
182
|
-
return lines.join("\n");
|
|
183
211
|
}
|
|
184
212
|
function tsvToCsvTransform(maxDataRows, onHeader) {
|
|
185
213
|
const state = createTsvStreamState();
|
|
@@ -309,6 +337,7 @@ export class DatasetsResource extends BaseResource {
|
|
|
309
337
|
*/
|
|
310
338
|
async preview(datasetName, opts) {
|
|
311
339
|
const maxRows = Math.max(1, Math.min(opts?.maxRows ?? 50, 500));
|
|
340
|
+
const timeoutMs = Math.max(1, opts?.timeoutMs ?? this.client.getRequestTimeoutMs());
|
|
312
341
|
const dsEnc = encodeURIComponent(datasetName);
|
|
313
342
|
const res = await this.client.stream(`/public/api/projects/${this.enc(opts?.projectKey)}/datasets/${dsEnc}/data/?format=tsv-excel-header&limit=${maxRows}`);
|
|
314
343
|
const onHeader = opts?.validateColumns
|
|
@@ -319,7 +348,9 @@ export class DatasetsResource extends BaseResource {
|
|
|
319
348
|
}
|
|
320
349
|
}
|
|
321
350
|
: undefined;
|
|
322
|
-
|
|
351
|
+
if (!res.body)
|
|
352
|
+
return "";
|
|
353
|
+
return collectPreviewCsv(res.body, maxRows, timeoutMs, onHeader);
|
|
323
354
|
}
|
|
324
355
|
/** Get dataset metadata (tags, custom fields, checklists). */
|
|
325
356
|
async metadata(datasetName, projectKey) {
|
|
@@ -1,20 +1,23 @@
|
|
|
1
1
|
import type { SqlQueryResponse, SqlQueryResult } from "../schemas.js";
|
|
2
2
|
import { BaseResource } from "./base.js";
|
|
3
|
+
type SqlQueryOptions = {
|
|
4
|
+
query: string;
|
|
5
|
+
connection?: string;
|
|
6
|
+
datasetFullName?: string;
|
|
7
|
+
database?: string;
|
|
8
|
+
preQueries?: string[];
|
|
9
|
+
postQueries?: string[];
|
|
10
|
+
type?: string;
|
|
11
|
+
projectKey?: string;
|
|
12
|
+
};
|
|
3
13
|
export declare class SqlResource extends BaseResource {
|
|
14
|
+
private resolveOptionalProjectKey;
|
|
4
15
|
/**
|
|
5
16
|
* Start a SQL query and return the queryId + schema.
|
|
6
17
|
* Specify either `connection` (run against a DB connection)
|
|
7
18
|
* or `datasetFullName` (run against a dataset's connection).
|
|
8
19
|
*/
|
|
9
|
-
startQuery(opts:
|
|
10
|
-
query: string;
|
|
11
|
-
connection?: string;
|
|
12
|
-
datasetFullName?: string;
|
|
13
|
-
database?: string;
|
|
14
|
-
preQueries?: string[];
|
|
15
|
-
postQueries?: string[];
|
|
16
|
-
type?: string;
|
|
17
|
-
}): Promise<SqlQueryResult>;
|
|
20
|
+
startQuery(opts: SqlQueryOptions): Promise<SqlQueryResult>;
|
|
18
21
|
/**
|
|
19
22
|
* Stream results of a started query as parsed JSON (array of arrays).
|
|
20
23
|
*/
|
|
@@ -24,17 +27,12 @@ export declare class SqlResource extends BaseResource {
|
|
|
24
27
|
* Throws on failure.
|
|
25
28
|
*/
|
|
26
29
|
finishStreaming(queryId: string): Promise<void>;
|
|
30
|
+
private executeQuery;
|
|
31
|
+
private resolveDatasetQueryFallback;
|
|
27
32
|
/**
|
|
28
33
|
* Execute a SQL query end-to-end: start, stream all rows, verify, return combined result.
|
|
29
34
|
* This is the primary method most callers want.
|
|
30
35
|
*/
|
|
31
|
-
query(opts:
|
|
32
|
-
query: string;
|
|
33
|
-
connection?: string;
|
|
34
|
-
datasetFullName?: string;
|
|
35
|
-
database?: string;
|
|
36
|
-
preQueries?: string[];
|
|
37
|
-
postQueries?: string[];
|
|
38
|
-
type?: string;
|
|
39
|
-
}): Promise<SqlQueryResponse>;
|
|
36
|
+
query(opts: SqlQueryOptions): Promise<SqlQueryResponse>;
|
|
40
37
|
}
|
|
38
|
+
export {};
|
|
@@ -13,7 +13,37 @@ function buildUnsupportedSqlDatasetConnectionMessage(datasetFullName) {
|
|
|
13
13
|
: "This query uses a connection that DSS does not support for direct SQL queries.";
|
|
14
14
|
return `${subject} Use --connection with a SQL-compatible connection instead.`;
|
|
15
15
|
}
|
|
16
|
+
function asRecord(value) {
|
|
17
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
18
|
+
return undefined;
|
|
19
|
+
return value;
|
|
20
|
+
}
|
|
21
|
+
function asString(value) {
|
|
22
|
+
if (typeof value !== "string")
|
|
23
|
+
return undefined;
|
|
24
|
+
const trimmed = value.trim();
|
|
25
|
+
return trimmed.length > 0 ? trimmed : undefined;
|
|
26
|
+
}
|
|
27
|
+
function splitDatasetIdentifier(datasetFullName, fallbackProjectKey) {
|
|
28
|
+
const trimmed = datasetFullName.trim();
|
|
29
|
+
const dotIndex = trimmed.indexOf(".");
|
|
30
|
+
if (dotIndex <= 0) {
|
|
31
|
+
return { datasetName: trimmed, projectKey: fallbackProjectKey, };
|
|
32
|
+
}
|
|
33
|
+
return {
|
|
34
|
+
projectKey: trimmed.slice(0, dotIndex),
|
|
35
|
+
datasetName: trimmed.slice(dotIndex + 1),
|
|
36
|
+
};
|
|
37
|
+
}
|
|
16
38
|
export class SqlResource extends BaseResource {
|
|
39
|
+
resolveOptionalProjectKey(projectKey) {
|
|
40
|
+
try {
|
|
41
|
+
return this.resolveProjectKey(projectKey);
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
return undefined;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
17
47
|
/**
|
|
18
48
|
* Start a SQL query and return the queryId + schema.
|
|
19
49
|
* Specify either `connection` (run against a DB connection)
|
|
@@ -22,6 +52,7 @@ export class SqlResource extends BaseResource {
|
|
|
22
52
|
async startQuery(opts) {
|
|
23
53
|
return this.client.post("/public/api/sql/queries/", {
|
|
24
54
|
...opts,
|
|
55
|
+
projectKey: opts.projectKey ?? this.resolveOptionalProjectKey(opts.projectKey),
|
|
25
56
|
type: opts.type ?? "sql",
|
|
26
57
|
});
|
|
27
58
|
}
|
|
@@ -44,6 +75,39 @@ export class SqlResource extends BaseResource {
|
|
|
44
75
|
throw new Error(`SQL query ${queryId} failed: ${text}`);
|
|
45
76
|
}
|
|
46
77
|
}
|
|
78
|
+
async executeQuery(opts) {
|
|
79
|
+
const { queryId, schema, } = await this.startQuery(opts);
|
|
80
|
+
const rows = await this.streamResults(queryId);
|
|
81
|
+
await this.finishStreaming(queryId);
|
|
82
|
+
return { queryId, schema, rows, };
|
|
83
|
+
}
|
|
84
|
+
async resolveDatasetQueryFallback(opts) {
|
|
85
|
+
const datasetFullName = opts.datasetFullName;
|
|
86
|
+
if (!datasetFullName)
|
|
87
|
+
return null;
|
|
88
|
+
try {
|
|
89
|
+
const identifier = splitDatasetIdentifier(datasetFullName, opts.projectKey);
|
|
90
|
+
const projectKey = identifier.projectKey
|
|
91
|
+
? identifier.projectKey
|
|
92
|
+
: this.resolveProjectKey(opts.projectKey);
|
|
93
|
+
const dsEnc = encodeURIComponent(identifier.datasetName);
|
|
94
|
+
const raw = await this.client.get(`/public/api/projects/${encodeURIComponent(projectKey)}/datasets/${dsEnc}`);
|
|
95
|
+
const params = asRecord(raw.params);
|
|
96
|
+
const connection = asString(params?.connection);
|
|
97
|
+
if (!connection)
|
|
98
|
+
return null;
|
|
99
|
+
return {
|
|
100
|
+
...opts,
|
|
101
|
+
connection,
|
|
102
|
+
datasetFullName: undefined,
|
|
103
|
+
database: opts.database ?? asString(params?.schema) ?? asString(params?.catalog),
|
|
104
|
+
projectKey,
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
catch {
|
|
108
|
+
return null;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
47
111
|
/**
|
|
48
112
|
* Execute a SQL query end-to-end: start, stream all rows, verify, return combined result.
|
|
49
113
|
* This is the primary method most callers want.
|
|
@@ -51,17 +115,18 @@ export class SqlResource extends BaseResource {
|
|
|
51
115
|
async query(opts) {
|
|
52
116
|
const queryOpts = { ...opts, type: opts.type ?? "sql", };
|
|
53
117
|
try {
|
|
54
|
-
|
|
55
|
-
const rows = await this.streamResults(queryId);
|
|
56
|
-
await this.finishStreaming(queryId);
|
|
57
|
-
return { queryId, schema, rows, };
|
|
118
|
+
return await this.executeQuery(queryOpts);
|
|
58
119
|
}
|
|
59
120
|
catch (error) {
|
|
60
121
|
if (!isUnsupportedSqlDatasetConnectionError(error))
|
|
61
122
|
throw error;
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
123
|
+
const retryOpts = await this.resolveDatasetQueryFallback(queryOpts);
|
|
124
|
+
if (!retryOpts) {
|
|
125
|
+
throw new Error(buildUnsupportedSqlDatasetConnectionMessage(queryOpts.datasetFullName), {
|
|
126
|
+
cause: error,
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
return this.executeQuery(retryOpts);
|
|
65
130
|
}
|
|
66
131
|
}
|
|
67
132
|
}
|
package/dist/src/skill.js
CHANGED
|
@@ -98,7 +98,9 @@ Use \`dss <resource> --help\` to see all actions and flags for any resource.
|
|
|
98
98
|
-v, --verbose log HTTP requests to stderr
|
|
99
99
|
--project-key KEY override default project for any command
|
|
100
100
|
--timeout MS request timeout (default: 30000)
|
|
101
|
-
--
|
|
101
|
+
--insecure disable TLS certificate verification
|
|
102
|
+
--ca-cert PATH trust an extra PEM CA bundle
|
|
103
|
+
--stdin read command input from stdin (JSON or SQL, depending on command)
|
|
102
104
|
\`\`\`
|
|
103
105
|
|
|
104
106
|
## Gotchas
|