dataiku-sdk 3.1.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +253 -38
- package/dist/build-metadata.json +1 -1
- package/dist/packages/types/src/index.d.ts +102 -0
- package/dist/packages/types/src/index.js +76 -0
- package/dist/src/auth.js +1 -1
- package/dist/src/cli/coerce.js +11 -2
- package/dist/src/cli/commands/analysis.js +4 -3
- package/dist/src/cli/commands/api-service.js +5 -3
- package/dist/src/cli/commands/app.js +61 -18
- package/dist/src/cli/commands/bundle.js +3 -2
- package/dist/src/cli/commands/code-env.js +23 -16
- package/dist/src/cli/commands/connection.d.ts +9 -0
- package/dist/src/cli/commands/connection.js +267 -1
- package/dist/src/cli/commands/dashboard.js +7 -6
- package/dist/src/cli/commands/data-collection.d.ts +2 -0
- package/dist/src/cli/commands/data-collection.js +164 -0
- package/dist/src/cli/commands/data-quality.js +9 -8
- package/dist/src/cli/commands/dataset.js +170 -40
- package/dist/src/cli/commands/doctor.js +4 -2
- package/dist/src/cli/commands/flow-zone.js +9 -8
- package/dist/src/cli/commands/folder.js +9 -8
- package/dist/src/cli/commands/future.js +4 -3
- package/dist/src/cli/commands/group.d.ts +2 -0
- package/dist/src/cli/commands/group.js +109 -0
- package/dist/src/cli/commands/index.js +151 -78
- package/dist/src/cli/commands/insight.js +8 -7
- package/dist/src/cli/commands/job.js +4 -3
- package/dist/src/cli/commands/knowledge-bank.d.ts +2 -0
- package/dist/src/cli/commands/knowledge-bank.js +80 -0
- package/dist/src/cli/commands/llm.d.ts +2 -0
- package/dist/src/cli/commands/llm.js +93 -0
- package/dist/src/cli/commands/macro.d.ts +2 -0
- package/dist/src/cli/commands/macro.js +221 -0
- package/dist/src/cli/commands/meaning.js +5 -4
- package/dist/src/cli/commands/ml-task.js +21 -8
- package/dist/src/cli/commands/model-evaluation-store.js +4 -3
- package/dist/src/cli/commands/notebook.js +17 -12
- package/dist/src/cli/commands/plugin.d.ts +2 -0
- package/dist/src/cli/commands/plugin.js +799 -0
- package/dist/src/cli/commands/project-folder.d.ts +2 -0
- package/dist/src/cli/commands/project-folder.js +197 -0
- package/dist/src/cli/commands/project-git.js +14 -54
- package/dist/src/cli/commands/project-library.js +7 -6
- package/dist/src/cli/commands/project.js +55 -8
- package/dist/src/cli/commands/recipe.js +50 -14
- package/dist/src/cli/commands/saved-model.d.ts +5 -0
- package/dist/src/cli/commands/saved-model.js +392 -4
- package/dist/src/cli/commands/scenario.js +147 -9
- package/dist/src/cli/commands/sql.js +3 -2
- package/dist/src/cli/commands/statistics.js +7 -6
- package/dist/src/cli/commands/user.d.ts +8 -0
- package/dist/src/cli/commands/user.js +296 -0
- package/dist/src/cli/commands/variable.js +2 -1
- package/dist/src/cli/commands/webapp.js +2 -1
- package/dist/src/cli/commands/wiki.js +7 -6
- package/dist/src/cli/contract.d.ts +2 -3
- package/dist/src/cli/contract.js +1167 -163
- package/dist/src/cli/flags.d.ts +4 -0
- package/dist/src/cli/flags.js +50 -2
- package/dist/src/cli/helpers/app-successor.d.ts +19 -4
- package/dist/src/cli/helpers/app-successor.js +90 -30
- package/dist/src/cli/helpers/cleanup.js +3 -1
- package/dist/src/cli/helpers/flow-zone.js +1 -2
- package/dist/src/cli/output.d.ts +1 -1
- package/dist/src/cli/output.js +23 -10
- package/dist/src/cli/usage.d.ts +1 -0
- package/dist/src/cli/usage.js +47 -1
- package/dist/src/cli.js +225 -95
- package/dist/src/client.d.ts +86 -6
- package/dist/src/client.js +263 -50
- package/dist/src/errors.d.ts +3 -0
- package/dist/src/errors.js +4 -0
- package/dist/src/index.d.ts +16 -7
- package/dist/src/index.js +11 -2
- package/dist/src/resources/analyses.js +1 -8
- package/dist/src/resources/applications.d.ts +8 -6
- package/dist/src/resources/applications.js +21 -21
- package/dist/src/resources/base.d.ts +4 -0
- package/dist/src/resources/base.js +15 -0
- package/dist/src/resources/bundles.d.ts +9 -4
- package/dist/src/resources/bundles.js +10 -5
- package/dist/src/resources/connections.d.ts +78 -1
- package/dist/src/resources/connections.js +132 -1
- package/dist/src/resources/data-collections.d.ts +118 -0
- package/dist/src/resources/data-collections.js +49 -0
- package/dist/src/resources/datasets.d.ts +69 -0
- package/dist/src/resources/datasets.js +99 -1
- package/dist/src/resources/folders.js +2 -1
- package/dist/src/resources/futures.d.ts +4 -1
- package/dist/src/resources/futures.js +38 -6
- package/dist/src/resources/groups.d.ts +47 -0
- package/dist/src/resources/groups.js +56 -0
- package/dist/src/resources/jobs.d.ts +8 -13
- package/dist/src/resources/jobs.js +41 -16
- package/dist/src/resources/knowledge-banks.d.ts +58 -0
- package/dist/src/resources/knowledge-banks.js +61 -0
- package/dist/src/resources/llms.d.ts +92 -0
- package/dist/src/resources/llms.js +62 -0
- package/dist/src/resources/macros.d.ts +121 -0
- package/dist/src/resources/macros.js +231 -0
- package/dist/src/resources/ml-tasks.d.ts +37 -2
- package/dist/src/resources/ml-tasks.js +100 -20
- package/dist/src/resources/model-evaluation-stores.js +1 -8
- package/dist/src/resources/notebooks.d.ts +43 -7
- package/dist/src/resources/notebooks.js +53 -13
- package/dist/src/resources/plugins.d.ts +265 -0
- package/dist/src/resources/plugins.js +458 -0
- package/dist/src/resources/project-folders.d.ts +71 -0
- package/dist/src/resources/project-folders.js +57 -0
- package/dist/src/resources/project-git.d.ts +6 -0
- package/dist/src/resources/project-git.js +69 -42
- package/dist/src/resources/project-library.d.ts +0 -1
- package/dist/src/resources/project-library.js +1 -17
- package/dist/src/resources/projects.d.ts +19 -3
- package/dist/src/resources/projects.js +30 -18
- package/dist/src/resources/recipes.d.ts +15 -1
- package/dist/src/resources/recipes.js +73 -6
- package/dist/src/resources/saved-models.d.ts +134 -1
- package/dist/src/resources/saved-models.js +214 -6
- package/dist/src/resources/scenarios.d.ts +105 -0
- package/dist/src/resources/scenarios.js +232 -106
- package/dist/src/resources/sql.js +5 -1
- package/dist/src/resources/users.d.ts +129 -0
- package/dist/src/resources/users.js +128 -0
- package/dist/src/resources/webapps.d.ts +15 -2
- package/dist/src/resources/webapps.js +18 -4
- package/dist/src/schemas.d.ts +2 -2
- package/dist/src/schemas.js +1 -1
- package/dist/src/skill.d.ts +13 -4
- package/dist/src/skill.js +75 -24
- package/dist/src/utils/deep-merge.d.ts +1 -1
- package/dist/src/utils/polling.d.ts +20 -0
- package/dist/src/utils/polling.js +27 -0
- package/dist/src/utils/response-file.d.ts +2 -0
- package/dist/src/utils/response-file.js +34 -0
- package/dist/src/utils/secret-sanitize.d.ts +54 -0
- package/dist/src/utils/secret-sanitize.js +101 -0
- package/package.json +8 -5
- package/packages/types/dist/index.d.ts +102 -0
- package/packages/types/dist/index.js +76 -0
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +47 -225
- package/skills/dataiku-dss/references/app-releases.md +28 -0
- package/skills/dataiku-dss/references/authentication.md +39 -0
- package/skills/dataiku-dss/references/coding.md +8 -0
- package/skills/dataiku-dss/references/discovery.md +30 -0
- package/skills/dataiku-dss/references/flow-maps.md +12 -0
- package/skills/dataiku-dss/references/mutations.md +33 -0
- package/skills/dataiku-dss/references/troubleshooting.md +29 -0
|
@@ -1,4 +1,81 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { sanitizeErrorSecrets, sanitizeSecrets, } from "../../utils/secret-sanitize.js";
|
|
2
|
+
import { requiredJsonInput, } from "../coerce.js";
|
|
3
|
+
import { executionMode, } from "../flags.js";
|
|
4
|
+
import { readIfExists, skipResult, } from "../output.js";
|
|
5
|
+
import { requireArgs, UsageError, } from "../usage.js";
|
|
6
|
+
/**
|
|
7
|
+
* Keys of a connection payload that carry credentials. CLI plan/error/output
|
|
8
|
+
* layers replace these with `[redacted]`; the SDK resource layer forwards
|
|
9
|
+
* values verbatim to DSS (over TLS) and returns server responses unmodified.
|
|
10
|
+
* Keys are matched after stripping `-`/`_` and lowercasing. The connection
|
|
11
|
+
* policy is deliberately broader than the user/project-git tables.
|
|
12
|
+
*/
|
|
13
|
+
const SENSITIVE_CONNECTION_KEYS = {
|
|
14
|
+
accesstoken: true,
|
|
15
|
+
accesskey: true,
|
|
16
|
+
apikey: true,
|
|
17
|
+
apitoken: true,
|
|
18
|
+
apisecret: true,
|
|
19
|
+
authorization: true,
|
|
20
|
+
clientsecret: true,
|
|
21
|
+
credential: true,
|
|
22
|
+
credentials: true,
|
|
23
|
+
password: true,
|
|
24
|
+
privatekey: true,
|
|
25
|
+
refreshtoken: true,
|
|
26
|
+
secret: true,
|
|
27
|
+
secretkey: true,
|
|
28
|
+
sharesecret: true,
|
|
29
|
+
token: true,
|
|
30
|
+
};
|
|
31
|
+
export function sanitizeConnectionSecrets(value, secrets = []) {
|
|
32
|
+
return sanitizeSecrets(value, { sensitiveKeys: SENSITIVE_CONNECTION_KEYS, secrets, });
|
|
33
|
+
}
|
|
34
|
+
/** Extract exact secret values (e.g. `params.password`) for error scrubbing. */
|
|
35
|
+
export function connectionSecretsFromBody(body) {
|
|
36
|
+
if (body === undefined || body === null || typeof body !== "object" || Array.isArray(body)) {
|
|
37
|
+
return [];
|
|
38
|
+
}
|
|
39
|
+
const found = [];
|
|
40
|
+
const visit = (value) => {
|
|
41
|
+
if (Array.isArray(value)) {
|
|
42
|
+
value.forEach(visit);
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
if (value !== null && typeof value === "object") {
|
|
46
|
+
for (const [key, item,] of Object.entries(value)) {
|
|
47
|
+
const normalizedKey = key.replace(/[-_]/g, "").toLowerCase();
|
|
48
|
+
if (SENSITIVE_CONNECTION_KEYS[normalizedKey] === true && typeof item === "string") {
|
|
49
|
+
found.push(item);
|
|
50
|
+
}
|
|
51
|
+
else {
|
|
52
|
+
visit(item);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
visit(body);
|
|
58
|
+
return found;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* DSS validation errors can echo the submitted connection body (including
|
|
62
|
+
* credentials). Scrub error text and body before it reaches stderr or a
|
|
63
|
+
* structured failure.
|
|
64
|
+
*/
|
|
65
|
+
export function sanitizeConnectionError(error, secrets) {
|
|
66
|
+
return sanitizeErrorSecrets(error, { sensitiveKeys: SENSITIVE_CONNECTION_KEYS, secrets, });
|
|
67
|
+
}
|
|
68
|
+
function requiredConnectionJson(flags, usage) {
|
|
69
|
+
return requiredJsonInput(flags, `--data, --data-file, or --stdin is required (connection definition). Usage: ${usage}`);
|
|
70
|
+
}
|
|
71
|
+
/** Connection identifiers must be non-empty before any request or dry-run preview. */
|
|
72
|
+
function requireConnectionName(value, usage) {
|
|
73
|
+
const trimmed = value?.trim() ?? "";
|
|
74
|
+
if (trimmed.length === 0) {
|
|
75
|
+
throw new UsageError(`connection name must be a non-empty string.\nUsage: ${usage}`, "validation_failed");
|
|
76
|
+
}
|
|
77
|
+
return trimmed;
|
|
78
|
+
}
|
|
2
79
|
export const connectionCommands = {
|
|
3
80
|
list: {
|
|
4
81
|
handler: (c, _a, f) => c.connections.list({
|
|
@@ -17,6 +94,118 @@ export const connectionCommands = {
|
|
|
17
94
|
description: "List connections with inferred types and metadata.",
|
|
18
95
|
examples: ["dss connection infer", "dss connection infer --mode rich",],
|
|
19
96
|
},
|
|
97
|
+
get: {
|
|
98
|
+
handler: async (c, a) => {
|
|
99
|
+
const usage = "dss connection get <name>";
|
|
100
|
+
requireArgs(a, 1, usage);
|
|
101
|
+
const details = await c.connections.adminGet(requireConnectionName(a[0], usage));
|
|
102
|
+
return sanitizeConnectionSecrets(details);
|
|
103
|
+
},
|
|
104
|
+
usage: "dss connection get <name>",
|
|
105
|
+
description: "Get a connection by name (admin). Credential-bearing params (e.g. params.password) are redacted in output.",
|
|
106
|
+
examples: ["dss connection get postgres",],
|
|
107
|
+
},
|
|
108
|
+
create: {
|
|
109
|
+
handler: async (c, _a, f) => {
|
|
110
|
+
const usage = "dss connection create (--data JSON|--data-file PATH|--stdin)";
|
|
111
|
+
const body = requiredConnectionJson(f, usage);
|
|
112
|
+
if (executionMode(f).dryRun) {
|
|
113
|
+
return {
|
|
114
|
+
dryRun: true,
|
|
115
|
+
action: "create",
|
|
116
|
+
resource: "connection",
|
|
117
|
+
connectionName: typeof body.name === "string" ? body.name : undefined,
|
|
118
|
+
method: "POST",
|
|
119
|
+
endpoint: "/public/api/admin/connections",
|
|
120
|
+
payload: sanitizeConnectionSecrets(body),
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
try {
|
|
124
|
+
return await c.connections.adminCreate(body);
|
|
125
|
+
}
|
|
126
|
+
catch (error) {
|
|
127
|
+
throw sanitizeConnectionError(error, connectionSecretsFromBody(body));
|
|
128
|
+
}
|
|
129
|
+
},
|
|
130
|
+
usage: "dss connection create (--data JSON|--data-file PATH|--stdin) [--dry-run]",
|
|
131
|
+
description: 'Create a connection (admin). Body is the Connection definition, e.g. {"name":"new-connection","type":"PostgreSQL","params":{"host":"...","user":"...","password":"..."}}. Secrets are never echoed in errors or output.',
|
|
132
|
+
examples: [
|
|
133
|
+
`dss connection create --data '{"name":"new-connection","type":"PostgreSQL","params":{"db":"dbname","user":"myuser","password":"s3cret"}}'`,
|
|
134
|
+
],
|
|
135
|
+
},
|
|
136
|
+
update: {
|
|
137
|
+
handler: async (c, a, f) => {
|
|
138
|
+
const usage = "dss connection update <name> (--data JSON|--data-file PATH|--stdin) [--dry-run]";
|
|
139
|
+
requireArgs(a, 1, usage);
|
|
140
|
+
const name = requireConnectionName(a[0], usage);
|
|
141
|
+
const body = requiredConnectionJson(f, usage);
|
|
142
|
+
const secrets = connectionSecretsFromBody(body);
|
|
143
|
+
if (executionMode(f).dryRun) {
|
|
144
|
+
return {
|
|
145
|
+
dryRun: true,
|
|
146
|
+
action: "update",
|
|
147
|
+
resource: "connection",
|
|
148
|
+
name,
|
|
149
|
+
method: "PUT",
|
|
150
|
+
endpoint: `/public/api/admin/connections/${encodeURIComponent(name)}`,
|
|
151
|
+
payload: sanitizeConnectionSecrets(body),
|
|
152
|
+
note: "PUT replaces the whole Connection object; the body must come from `dss connection get`. No GET was issued in dry-run.",
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
try {
|
|
156
|
+
const result = await c.connections.adminUpdate(name, body);
|
|
157
|
+
return sanitizeConnectionSecrets(result, secrets);
|
|
158
|
+
}
|
|
159
|
+
catch (error) {
|
|
160
|
+
throw sanitizeConnectionError(error, secrets);
|
|
161
|
+
}
|
|
162
|
+
},
|
|
163
|
+
usage: "dss connection update <name> (--data JSON|--data-file PATH|--stdin) [--dry-run]",
|
|
164
|
+
description: "Update a connection (admin). The body MUST be the Connection object obtained from `dss connection get` (PUT semantics); type and name cannot be modified. Use --dry-run to preview the merged result without writing.",
|
|
165
|
+
examples: [
|
|
166
|
+
`dss connection update postgres --data '{"type":"PostgreSQL","name":"postgres","allowWrite":true,"params":{"db":"dbname","user":"myuser","password":"s3cret"}}'`,
|
|
167
|
+
],
|
|
168
|
+
},
|
|
169
|
+
delete: {
|
|
170
|
+
handler: async (c, a, f) => {
|
|
171
|
+
const usage = "dss connection delete <name> [--if-exists] [--dry-run]";
|
|
172
|
+
requireArgs(a, 1, usage);
|
|
173
|
+
const name = requireConnectionName(a[0], usage);
|
|
174
|
+
if (executionMode(f).dryRun) {
|
|
175
|
+
return {
|
|
176
|
+
dryRun: true,
|
|
177
|
+
action: "delete",
|
|
178
|
+
resource: "connection",
|
|
179
|
+
name,
|
|
180
|
+
method: "DELETE",
|
|
181
|
+
endpoint: `/public/api/admin/connections/${encodeURIComponent(name)}`,
|
|
182
|
+
...(f["if-exists"] === true
|
|
183
|
+
? { note: "Existence not checked in dry-run; live --if-exists issues a GET probe first.", }
|
|
184
|
+
: {}),
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
if (f["if-exists"] === true) {
|
|
188
|
+
const current = await readIfExists(() => c.connections.adminGet(name));
|
|
189
|
+
if (!current)
|
|
190
|
+
return skipResult("connection", name, "missing");
|
|
191
|
+
}
|
|
192
|
+
return c.connections.adminDelete(name);
|
|
193
|
+
},
|
|
194
|
+
usage: "dss connection delete <name> [--if-exists] [--dry-run]",
|
|
195
|
+
description: "Delete a connection (admin). Destructive: per the DSS docs no check is performed that the connection is not in use by a dataset.",
|
|
196
|
+
examples: ["dss connection delete old-conn", "dss connection delete old-conn --if-exists",],
|
|
197
|
+
},
|
|
198
|
+
test: {
|
|
199
|
+
handler: async (c, a) => {
|
|
200
|
+
const usage = "dss connection test <name>";
|
|
201
|
+
requireArgs(a, 1, usage);
|
|
202
|
+
const result = await c.connections.adminTest(requireConnectionName(a[0], usage));
|
|
203
|
+
return sanitizeConnectionSecrets(result);
|
|
204
|
+
},
|
|
205
|
+
usage: "dss connection test <name>",
|
|
206
|
+
description: "Test whether a connection is available (uses the same GET /connections/{name}/test route as the official Python client; returns connectionOK, errors when testing is unsupported for the type). Credential-bearing fields are redacted in output.",
|
|
207
|
+
examples: ["dss connection test postgres",],
|
|
208
|
+
},
|
|
20
209
|
schemas: {
|
|
21
210
|
handler: (c, _a, f) => {
|
|
22
211
|
const connection = f["connection"];
|
|
@@ -51,4 +240,81 @@ export const connectionCommands = {
|
|
|
51
240
|
"dss connection tables --connection ATHENA_CONN --schema analytics --project-key MYPROJ",
|
|
52
241
|
],
|
|
53
242
|
},
|
|
243
|
+
"prepare-import": {
|
|
244
|
+
handler: async (c, _a, f) => {
|
|
245
|
+
const usage = "dss connection prepare-import (--data JSON|--data-file PATH|--stdin) [--project-key KEY]";
|
|
246
|
+
const body = requiredJsonInput(f, `--data, --data-file, or --stdin is required (tables-import request). Usage: ${usage}`);
|
|
247
|
+
const keys = body["keys"];
|
|
248
|
+
if (!Array.isArray(keys) || keys.length === 0) {
|
|
249
|
+
throw new UsageError('Body must include a non-empty keys array (e.g. [{"connectionName":"postgres","name":"my_table"}]).', "validation_failed", `Usage: ${usage}`);
|
|
250
|
+
}
|
|
251
|
+
for (const key of keys) {
|
|
252
|
+
const k = key;
|
|
253
|
+
const connectionName = k["connectionName"];
|
|
254
|
+
const name = k["name"];
|
|
255
|
+
if (typeof connectionName !== "string" || connectionName.trim().length === 0
|
|
256
|
+
|| typeof name !== "string" || name.trim().length === 0) {
|
|
257
|
+
throw new UsageError('Each key must carry non-empty string connectionName and name (e.g. {"connectionName":"postgres","name":"my_table"}).', "validation_failed", `Usage: ${usage}`);
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
if (executionMode(f).dryRun) {
|
|
261
|
+
return {
|
|
262
|
+
dryRun: true,
|
|
263
|
+
action: "prepare-import",
|
|
264
|
+
resource: "connection",
|
|
265
|
+
keyCount: keys.length,
|
|
266
|
+
method: "POST",
|
|
267
|
+
endpoint: `/public/api/projects/${encodeURIComponent(c.resolveProjectKey(f["project-key"]))}/datasets/tables-import/actions/prepare-from-keys`,
|
|
268
|
+
payload: { keys, },
|
|
269
|
+
};
|
|
270
|
+
}
|
|
271
|
+
return c.connections.prepareTablesImport({
|
|
272
|
+
keys: keys,
|
|
273
|
+
projectKey: f["project-key"],
|
|
274
|
+
});
|
|
275
|
+
},
|
|
276
|
+
usage: "dss connection prepare-import (--data JSON|--data-file PATH|--stdin) [--project-key KEY] [--dry-run]",
|
|
277
|
+
description: 'Prepare the import of selected SQL or Hive tables (WRITE_CONF). Body is {"keys":[...]} per the DSS tables-import docs. Returns a DSS future; poll with `dss future wait <jobId>`.',
|
|
278
|
+
examples: [
|
|
279
|
+
`dss connection prepare-import --data '{"keys":[{"connectionName":"postgres","name":"my_table"}]}' --project-key MYPROJ`,
|
|
280
|
+
],
|
|
281
|
+
},
|
|
282
|
+
"execute-import": {
|
|
283
|
+
handler: async (c, _a, f) => {
|
|
284
|
+
const usage = "dss connection execute-import (--data JSON|--data-file PATH|--stdin) [--project-key KEY]";
|
|
285
|
+
const body = requiredJsonInput(f, `--data, --data-file, or --stdin is required (tables-import request). Usage: ${usage}`);
|
|
286
|
+
const sql = body["sqlImportCandidates"];
|
|
287
|
+
const hive = body["hiveImportCandidates"];
|
|
288
|
+
const hasSql = Array.isArray(sql) && sql.length > 0;
|
|
289
|
+
const hasHive = Array.isArray(hive) && hive.length > 0;
|
|
290
|
+
if (!hasSql && !hasHive) {
|
|
291
|
+
throw new UsageError('Body must include sqlImportCandidates and/or hiveImportCandidates arrays (e.g. {"sqlImportCandidates":[{"connectionName":"pgsql","table":"t","checked":false,"datasetName":"imported","existingDatasetsNames":[]}]}).', "validation_failed", `Usage: ${usage}`);
|
|
292
|
+
}
|
|
293
|
+
if (executionMode(f).dryRun) {
|
|
294
|
+
return {
|
|
295
|
+
dryRun: true,
|
|
296
|
+
action: "execute-import",
|
|
297
|
+
resource: "connection",
|
|
298
|
+
sqlCandidates: hasSql ? sql.length : undefined,
|
|
299
|
+
hiveCandidates: hasHive ? hive.length : undefined,
|
|
300
|
+
method: "POST",
|
|
301
|
+
endpoint: `/public/api/projects/${encodeURIComponent(c.resolveProjectKey(f["project-key"]))}/datasets/tables-import/actions/execute-from-candidates`,
|
|
302
|
+
payload: {
|
|
303
|
+
...(hasSql ? { sqlImportCandidates: sql, } : {}),
|
|
304
|
+
...(hasHive ? { hiveImportCandidates: hive, } : {}),
|
|
305
|
+
},
|
|
306
|
+
};
|
|
307
|
+
}
|
|
308
|
+
return c.connections.executeTablesImport({
|
|
309
|
+
sqlImportCandidates: hasSql ? sql : undefined,
|
|
310
|
+
hiveImportCandidates: hasHive ? hive : undefined,
|
|
311
|
+
projectKey: f["project-key"],
|
|
312
|
+
});
|
|
313
|
+
},
|
|
314
|
+
usage: "dss connection execute-import (--data JSON|--data-file PATH|--stdin) [--project-key KEY] [--dry-run]",
|
|
315
|
+
description: "Perform an import from SQL/Hive table candidates (WRITE_CONF). Body carries sqlImportCandidates and/or hiveImportCandidates per the DSS tables-import docs. Returns a DSS future.",
|
|
316
|
+
examples: [
|
|
317
|
+
`dss connection execute-import --data '{"sqlImportCandidates":[{"connectionName":"pgsql","table":"my_table","checked":false,"datasetName":"imported_from_db","existingDatasetsNames":[]}]}' --project-key MYPROJ`,
|
|
318
|
+
],
|
|
319
|
+
},
|
|
54
320
|
};
|
|
@@ -2,6 +2,7 @@ import * as fs from "node:fs";
|
|
|
2
2
|
import { ClientValidationError, } from "../../errors.js";
|
|
3
3
|
import { deepMerge, } from "../../utils/deep-merge.js";
|
|
4
4
|
import { jsonInput, parseBooleanOption, } from "../coerce.js";
|
|
5
|
+
import { executionMode, } from "../flags.js";
|
|
5
6
|
import { readIfExists, skipResult, } from "../output.js";
|
|
6
7
|
import { requireArgs, UsageError, } from "../usage.js";
|
|
7
8
|
export const dashboardCommands = {
|
|
@@ -106,12 +107,12 @@ export const dashboardCommands = {
|
|
|
106
107
|
if (listed !== undefined)
|
|
107
108
|
payload.listed = listed;
|
|
108
109
|
const pk = f["project-key"];
|
|
109
|
-
if (f["if-not-exists"] === true || f
|
|
110
|
+
if (f["if-not-exists"] === true || executionMode(f).dryRun) {
|
|
110
111
|
const existing = (await c.dashboards.list(pk)).find((dashboard) => dashboard.name === name);
|
|
111
|
-
if (existing && f["if-not-exists"] === true && f
|
|
112
|
+
if (existing && f["if-not-exists"] === true && !executionMode(f).dryRun) {
|
|
112
113
|
return skipResult("dashboard", existing.id, "exists", { current: existing, });
|
|
113
114
|
}
|
|
114
|
-
if (f
|
|
115
|
+
if (executionMode(f).dryRun) {
|
|
115
116
|
return {
|
|
116
117
|
dryRun: true,
|
|
117
118
|
action: "create",
|
|
@@ -146,7 +147,7 @@ export const dashboardCommands = {
|
|
|
146
147
|
if (name === undefined && listed === undefined && !data) {
|
|
147
148
|
throw new UsageError("--name, --listed, --data, --data-file, or --stdin is required.");
|
|
148
149
|
}
|
|
149
|
-
if (f
|
|
150
|
+
if (executionMode(f).dryRun) {
|
|
150
151
|
const current = await c.dashboards.get(a[0], f["project-key"]);
|
|
151
152
|
const next = deepMerge(current, data ?? {});
|
|
152
153
|
if (name !== undefined)
|
|
@@ -170,11 +171,11 @@ export const dashboardCommands = {
|
|
|
170
171
|
handler: async (c, a, f) => {
|
|
171
172
|
requireArgs(a, 1, "dss dashboard delete <id>");
|
|
172
173
|
const pk = f["project-key"];
|
|
173
|
-
if (f
|
|
174
|
+
if (executionMode(f).dryRun || f["if-exists"] === true) {
|
|
174
175
|
const current = await readIfExists(() => c.dashboards.get(a[0], pk));
|
|
175
176
|
if (!current)
|
|
176
177
|
return skipResult("dashboard", a[0], "missing");
|
|
177
|
-
if (f
|
|
178
|
+
if (executionMode(f).dryRun) {
|
|
178
179
|
return { dryRun: true, action: "delete", resource: "dashboard", id: a[0], current, };
|
|
179
180
|
}
|
|
180
181
|
}
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
import { requiredJsonInput, } from "../coerce.js";
|
|
2
|
+
import { executionMode, } from "../flags.js";
|
|
3
|
+
import { planResult, } from "../output.js";
|
|
4
|
+
import { requireArgs, UsageError, } from "../usage.js";
|
|
5
|
+
const PLAN_FAILURE_CODES = { usage: 1, error: 2, transient: 3, };
|
|
6
|
+
function collectionEndpoint(dataCollectionId, suffix = "") {
|
|
7
|
+
return `/public/api/data-collections/${encodeURIComponent(dataCollectionId)}${suffix}`;
|
|
8
|
+
}
|
|
9
|
+
export const dataCollectionCommands = {
|
|
10
|
+
list: {
|
|
11
|
+
handler: (c) => c.dataCollections.list(),
|
|
12
|
+
usage: "dss data-collection list",
|
|
13
|
+
description: "List data collections on which the API key has READ privilege.",
|
|
14
|
+
examples: ["dss data-collection list",],
|
|
15
|
+
},
|
|
16
|
+
get: {
|
|
17
|
+
handler: (c, a) => {
|
|
18
|
+
requireArgs(a, 1, "dss data-collection get <dataCollectionId>");
|
|
19
|
+
return c.dataCollections.get(a[0]);
|
|
20
|
+
},
|
|
21
|
+
usage: "dss data-collection get <dataCollectionId>",
|
|
22
|
+
description: "Get collection settings. Permissions are omitted for non-admins.",
|
|
23
|
+
examples: ["dss data-collection get OjVsTQ3O",],
|
|
24
|
+
},
|
|
25
|
+
create: {
|
|
26
|
+
handler: async (c, _a, f) => {
|
|
27
|
+
const body = requiredJsonInput(f, "--data, --data-file, or --stdin is required (collection definition).");
|
|
28
|
+
if (executionMode(f).dryRun) {
|
|
29
|
+
return planResult("data-collection", "create", {
|
|
30
|
+
asyncKind: "none",
|
|
31
|
+
method: "POST",
|
|
32
|
+
endpoint: "/public/api/data-collections/",
|
|
33
|
+
payload: body,
|
|
34
|
+
idempotency: "none",
|
|
35
|
+
exitCodesOnFailure: PLAN_FAILURE_CODES,
|
|
36
|
+
plannedAndDryRun: true,
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
return c.dataCollections.create(body);
|
|
40
|
+
},
|
|
41
|
+
usage: "dss data-collection create (--data JSON|--data-file PATH|--stdin) [--dry-run]",
|
|
42
|
+
description: "Create a collection (creator becomes admin; requires mayCreateDataCollections).",
|
|
43
|
+
examples: [
|
|
44
|
+
'dss data-collection create --data \'{"displayName":"My Collection"}\'',
|
|
45
|
+
],
|
|
46
|
+
},
|
|
47
|
+
"settings-set": {
|
|
48
|
+
handler: async (c, a, f) => {
|
|
49
|
+
requireArgs(a, 1, "dss data-collection settings-set <dataCollectionId> (--data JSON|--data-file PATH|--stdin) [--dry-run]");
|
|
50
|
+
const settings = requiredJsonInput(f, "--data, --data-file, or --stdin is required (collection settings).");
|
|
51
|
+
if (executionMode(f).dryRun) {
|
|
52
|
+
return planResult("data-collection", "settings-set", {
|
|
53
|
+
asyncKind: "none",
|
|
54
|
+
method: "PUT",
|
|
55
|
+
endpoint: collectionEndpoint(a[0]),
|
|
56
|
+
identifiers: { dataCollectionId: a[0], },
|
|
57
|
+
payload: settings,
|
|
58
|
+
idempotency: "none",
|
|
59
|
+
exitCodesOnFailure: PLAN_FAILURE_CODES,
|
|
60
|
+
plannedAndDryRun: true,
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
const updated = await c.dataCollections.update(a[0], settings);
|
|
64
|
+
return { updated: a[0], resource: "data-collection", ...updated, };
|
|
65
|
+
},
|
|
66
|
+
usage: "dss data-collection settings-set <dataCollectionId> (--data JSON|--data-file PATH|--stdin) [--dry-run]",
|
|
67
|
+
description: "Update collection settings; only send settings obtained through get. Admin required.",
|
|
68
|
+
examples: [
|
|
69
|
+
"dss data-collection settings-set OjVsTQ3O --data-file settings.json",
|
|
70
|
+
],
|
|
71
|
+
},
|
|
72
|
+
delete: {
|
|
73
|
+
validate: (a) => {
|
|
74
|
+
const usage = "dss data-collection delete <dataCollectionId> [--dry-run]";
|
|
75
|
+
requireArgs(a, 1, usage);
|
|
76
|
+
if (a[0].trim().length === 0) {
|
|
77
|
+
throw new UsageError(`<dataCollectionId> must be a non-empty identifier. Usage: ${usage}`);
|
|
78
|
+
}
|
|
79
|
+
},
|
|
80
|
+
handler: async (c, a, f) => {
|
|
81
|
+
if (executionMode(f).dryRun) {
|
|
82
|
+
return planResult("data-collection", "delete", {
|
|
83
|
+
asyncKind: "none",
|
|
84
|
+
method: "DELETE",
|
|
85
|
+
endpoint: collectionEndpoint(a[0]),
|
|
86
|
+
identifiers: { dataCollectionId: a[0], },
|
|
87
|
+
idempotency: "none",
|
|
88
|
+
exitCodesOnFailure: PLAN_FAILURE_CODES,
|
|
89
|
+
plannedAndDryRun: true,
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
await c.dataCollections.delete(a[0]);
|
|
93
|
+
return { deleted: a[0], resource: "data-collection", };
|
|
94
|
+
},
|
|
95
|
+
usage: "dss data-collection delete <dataCollectionId> [--dry-run]",
|
|
96
|
+
description: "Permanently delete a collection. Admin required.",
|
|
97
|
+
examples: ["dss data-collection delete OjVsTQ3O --dry-run",],
|
|
98
|
+
},
|
|
99
|
+
"list-objects": {
|
|
100
|
+
handler: (c, a) => {
|
|
101
|
+
requireArgs(a, 1, "dss data-collection list-objects <dataCollectionId>");
|
|
102
|
+
return c.dataCollections.listObjects(a[0]);
|
|
103
|
+
},
|
|
104
|
+
usage: "dss data-collection list-objects <dataCollectionId>",
|
|
105
|
+
description: "List the objects in a collection.",
|
|
106
|
+
examples: ["dss data-collection list-objects OjVsTQ3O",],
|
|
107
|
+
},
|
|
108
|
+
"add-object": {
|
|
109
|
+
handler: async (c, a, f) => {
|
|
110
|
+
const object = requiredJsonInput(f, "--data, --data-file, or --stdin is required (collection object reference).");
|
|
111
|
+
const collectionId = a[0]
|
|
112
|
+
?? (typeof object.dataCollectionId === "string" ? object.dataCollectionId : undefined);
|
|
113
|
+
if (!collectionId) {
|
|
114
|
+
throw new UsageError("Usage: dss data-collection add-object <dataCollectionId> (--data JSON|--data-file PATH|--stdin)");
|
|
115
|
+
}
|
|
116
|
+
const reference = { ...object, };
|
|
117
|
+
delete reference.dataCollectionId;
|
|
118
|
+
if (executionMode(f).dryRun) {
|
|
119
|
+
return planResult("data-collection", "add-object", {
|
|
120
|
+
asyncKind: "none",
|
|
121
|
+
method: "POST",
|
|
122
|
+
endpoint: `${collectionEndpoint(collectionId)}/objects`,
|
|
123
|
+
identifiers: { dataCollectionId: collectionId, },
|
|
124
|
+
payload: reference,
|
|
125
|
+
idempotency: "none",
|
|
126
|
+
exitCodesOnFailure: PLAN_FAILURE_CODES,
|
|
127
|
+
plannedAndDryRun: true,
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
await c.dataCollections.addObject(collectionId, reference);
|
|
131
|
+
return { added: reference.id, dataCollectionId: collectionId, resource: "data-collection", };
|
|
132
|
+
},
|
|
133
|
+
usage: "dss data-collection add-object <dataCollectionId> (--data JSON|--data-file PATH|--stdin) [--dry-run]",
|
|
134
|
+
description: "Add an object (e.g. {type:'DATASET',projectKey:'K',id:'name'}) to a collection. Contributor required.",
|
|
135
|
+
examples: [
|
|
136
|
+
'dss data-collection add-object OjVsTQ3O --data \'{"type":"DATASET","projectKey":"MYPROJECT","id":"orders"}\'',
|
|
137
|
+
],
|
|
138
|
+
},
|
|
139
|
+
"remove-dataset": {
|
|
140
|
+
handler: async (c, a, f) => {
|
|
141
|
+
requireArgs(a, 3, "dss data-collection remove-dataset <dataCollectionId> <projectKey> <datasetName> [--dry-run]");
|
|
142
|
+
if (executionMode(f).dryRun) {
|
|
143
|
+
return planResult("data-collection", "remove-dataset", {
|
|
144
|
+
asyncKind: "none",
|
|
145
|
+
method: "DELETE",
|
|
146
|
+
endpoint: `${collectionEndpoint(a[0])}/objects/dataset/${encodeURIComponent(a[1])}/${encodeURIComponent(a[2])}`,
|
|
147
|
+
identifiers: {
|
|
148
|
+
dataCollectionId: a[0],
|
|
149
|
+
projectKey: a[1],
|
|
150
|
+
datasetName: a[2],
|
|
151
|
+
},
|
|
152
|
+
idempotency: "none",
|
|
153
|
+
exitCodesOnFailure: PLAN_FAILURE_CODES,
|
|
154
|
+
plannedAndDryRun: true,
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
await c.dataCollections.removeDataset(a[0], a[1], a[2]);
|
|
158
|
+
return { removed: a[2], projectKey: a[1], dataCollectionId: a[0], resource: "data-collection", };
|
|
159
|
+
},
|
|
160
|
+
usage: "dss data-collection remove-dataset <dataCollectionId> <projectKey> <datasetName> [--dry-run]",
|
|
161
|
+
description: "Remove a dataset from a collection. Contributor required.",
|
|
162
|
+
examples: ["dss data-collection remove-dataset OjVsTQ3O MYPROJECT DATASET1",],
|
|
163
|
+
},
|
|
164
|
+
};
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { deepMerge, } from "../../utils/deep-merge.js";
|
|
2
2
|
import { num, parseBooleanOption, requiredJsonInput, } from "../coerce.js";
|
|
3
|
+
import { executionMode, } from "../flags.js";
|
|
3
4
|
import { encodedProjectEndpoint, readIfExists, skipResult, } from "../output.js";
|
|
4
5
|
import { requireArgs, UsageError, } from "../usage.js";
|
|
5
6
|
/**
|
|
@@ -52,20 +53,20 @@ export const dataQualityCommands = {
|
|
|
52
53
|
: typeof config.displayName === "string"
|
|
53
54
|
? config.displayName
|
|
54
55
|
: undefined;
|
|
55
|
-
if (f["if-not-exists"] === true || f
|
|
56
|
+
if (f["if-not-exists"] === true || executionMode(f).dryRun) {
|
|
56
57
|
const existing = identity
|
|
57
58
|
? (await c.dataQuality.listRules(a[0], pk)).find((rule) => rule.id === identity || rule.displayName === identity)
|
|
58
59
|
: undefined;
|
|
59
|
-
if (existing && f["if-not-exists"] === true && f
|
|
60
|
+
if (existing && f["if-not-exists"] === true && !executionMode(f).dryRun) {
|
|
60
61
|
return skipResult("data-quality", identity ?? existing.id, "exists", {
|
|
61
62
|
dataset: a[0],
|
|
62
63
|
current: existing,
|
|
63
64
|
});
|
|
64
65
|
}
|
|
65
|
-
if (f["if-not-exists"] === true && !identity && f
|
|
66
|
+
if (f["if-not-exists"] === true && !identity && !executionMode(f).dryRun) {
|
|
66
67
|
throw new UsageError("--if-not-exists requires rule id or displayName in the rule JSON.");
|
|
67
68
|
}
|
|
68
|
-
if (f
|
|
69
|
+
if (executionMode(f).dryRun) {
|
|
69
70
|
return {
|
|
70
71
|
dryRun: true,
|
|
71
72
|
action: "create-rule",
|
|
@@ -97,7 +98,7 @@ export const dataQualityCommands = {
|
|
|
97
98
|
handler: async (c, a, f) => {
|
|
98
99
|
requireArgs(a, 2, "dss data-quality update-rule <dataset> <rule-id> --data JSON");
|
|
99
100
|
const data = requiredJsonInput(f, "--data, --data-file, or --stdin is required.");
|
|
100
|
-
if (f
|
|
101
|
+
if (executionMode(f).dryRun) {
|
|
101
102
|
const current = await c.dataQuality.getRule(a[0], a[1], f["project-key"]);
|
|
102
103
|
const next = deepMerge(current, data);
|
|
103
104
|
return {
|
|
@@ -125,11 +126,11 @@ export const dataQualityCommands = {
|
|
|
125
126
|
handler: async (c, a, f) => {
|
|
126
127
|
requireArgs(a, 2, "dss data-quality delete-rule <dataset> <rule-id>");
|
|
127
128
|
const pk = f["project-key"];
|
|
128
|
-
if (f
|
|
129
|
+
if (executionMode(f).dryRun || f["if-exists"] === true) {
|
|
129
130
|
const current = await readIfExists(() => c.dataQuality.getRule(a[0], a[1], pk));
|
|
130
131
|
if (!current)
|
|
131
132
|
return skipResult("data-quality", a[1], "missing", { dataset: a[0], });
|
|
132
|
-
if (f
|
|
133
|
+
if (executionMode(f).dryRun) {
|
|
133
134
|
return {
|
|
134
135
|
dryRun: true,
|
|
135
136
|
action: "delete-rule",
|
|
@@ -248,7 +249,7 @@ export const dataQualityCommands = {
|
|
|
248
249
|
projectKey: pk,
|
|
249
250
|
timeoutMs: num(f["timeout"], "--timeout"),
|
|
250
251
|
};
|
|
251
|
-
if (f
|
|
252
|
+
if (executionMode(f).dryRun) {
|
|
252
253
|
const params = new URLSearchParams();
|
|
253
254
|
params.set("partition", options.partition?.trim() ? options.partition : "NP");
|
|
254
255
|
if (options.ruleId !== undefined)
|