dataiku-sdk 3.7.0 → 3.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/build-metadata.json +1 -1
- package/dist/packages/types/src/index.d.ts +40 -4
- package/dist/packages/types/src/index.js +22 -1
- package/dist/src/cli/coerce.d.ts +1 -0
- package/dist/src/cli/coerce.js +1 -1
- package/dist/src/cli/command-syntax.json +156 -0
- package/dist/src/cli/commands/dataset.js +44 -8
- package/dist/src/cli/commands/flow-zone.js +241 -46
- package/dist/src/cli/commands/job.js +52 -9
- package/dist/src/cli/commands/project.js +4 -3
- package/dist/src/cli/commands/recipe.d.ts +4 -0
- package/dist/src/cli/commands/recipe.js +38 -1
- package/dist/src/cli/commands/wiki.js +56 -0
- package/dist/src/cli/contract.js +13 -8
- package/dist/src/cli/doctor.js +2 -0
- package/dist/src/cli/flags.d.ts +7 -0
- package/dist/src/cli/flags.js +28 -2
- package/dist/src/cli/helpers/dataset.d.ts +12 -0
- package/dist/src/cli/helpers/dataset.js +36 -0
- package/dist/src/cli/helpers/flow-zone.d.ts +21 -0
- package/dist/src/cli/helpers/flow-zone.js +144 -4
- package/dist/src/cli/output-schemas.js +27 -3
- package/dist/src/cli/output.js +6 -0
- package/dist/src/cli/plans.js +58 -2
- package/dist/src/cli/project-map-output.d.ts +24 -0
- package/dist/src/cli/project-map-output.js +21 -0
- package/dist/src/cli/read-shapes.d.ts +14 -0
- package/dist/src/cli/read-shapes.js +114 -0
- package/dist/src/cli/result-shapes.d.ts +6 -0
- package/dist/src/cli/result-shapes.js +235 -0
- package/dist/src/cli/types.d.ts +2 -0
- package/dist/src/cli/version.d.ts +2 -2
- package/dist/src/cli/version.js +2 -2
- package/dist/src/cli.js +23 -20
- package/dist/src/errors.d.ts +11 -1
- package/dist/src/errors.js +18 -3
- package/dist/src/generated/action-output-schemas.json +224 -35
- package/dist/src/resources/dataset-create.d.ts +19 -0
- package/dist/src/resources/dataset-create.js +69 -0
- package/dist/src/resources/datasets.d.ts +21 -0
- package/dist/src/resources/datasets.js +61 -1
- package/dist/src/resources/flow-zones.d.ts +29 -3
- package/dist/src/resources/flow-zones.js +53 -3
- package/dist/src/resources/jobs.d.ts +36 -1
- package/dist/src/resources/jobs.js +111 -5
- package/dist/src/resources/projects.d.ts +9 -0
- package/dist/src/resources/projects.js +10 -3
- package/dist/src/resources/recipe-create.d.ts +17 -3
- package/dist/src/resources/recipe-create.js +149 -33
- package/dist/src/resources/recipes.js +76 -23
- package/dist/src/resources/sql.js +52 -13
- package/dist/src/resources/wiki.d.ts +47 -1
- package/dist/src/resources/wiki.js +89 -0
- package/dist/src/utils/flow-analysis.js +4 -0
- package/dist/src/utils/secret-sanitize.d.ts +11 -2
- package/dist/src/utils/secret-sanitize.js +23 -6
- package/package.json +4 -4
- package/plugin.json +1 -1
- package/skills/dataiku-dss/references/discovery.md +2 -2
- package/skills/dataiku-dss/references/flow-maps.md +2 -2
- package/skills/dataiku-dss/references/troubleshooting.md +3 -3
|
@@ -194,46 +194,158 @@ function processorStep(type, params) {
|
|
|
194
194
|
alwaysShowComment: false,
|
|
195
195
|
};
|
|
196
196
|
}
|
|
197
|
+
/** Prepare option groups, in the default step order. */
|
|
198
|
+
export const PREPARE_STEP_KINDS = [
|
|
199
|
+
"rename",
|
|
200
|
+
"fillEmpty",
|
|
201
|
+
"formula",
|
|
202
|
+
"filter",
|
|
203
|
+
"dropColumns",
|
|
204
|
+
"keepColumns",
|
|
205
|
+
];
|
|
206
|
+
function isPrepareStepKind(value) {
|
|
207
|
+
return PREPARE_STEP_KINDS.includes(value);
|
|
208
|
+
}
|
|
197
209
|
/**
|
|
198
|
-
*
|
|
199
|
-
*
|
|
200
|
-
* steps see renamed columns. Empty when no prepare option was given.
|
|
210
|
+
* Order the prepare option groups run in: `stepOrder` (the CLI passes the order
|
|
211
|
+
* the flags were typed in) first, then any group it leaves out in the default order.
|
|
201
212
|
*/
|
|
202
|
-
|
|
203
|
-
const
|
|
204
|
-
const
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
for (const raw of opts.fillEmpty ?? []) {
|
|
211
|
-
const [column, value,] = splitPair(raw, "--fill-empty");
|
|
212
|
-
steps.push(processorStep("FillEmptyWithValue", { appliesTo: "SINGLE_COLUMN", columns: [column,], value, }));
|
|
213
|
+
function prepareStepOrder(opts) {
|
|
214
|
+
const order = [];
|
|
215
|
+
for (const kind of opts.stepOrder ?? []) {
|
|
216
|
+
if (!isPrepareStepKind(kind)) {
|
|
217
|
+
throw new ClientValidationError(`Unknown stepOrder entry "${kind}"; expected one of ${PREPARE_STEP_KINDS.join(", ")}.`, "invalid_flag_value");
|
|
218
|
+
}
|
|
219
|
+
if (!order.includes(kind))
|
|
220
|
+
order.push(kind);
|
|
213
221
|
}
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
222
|
+
for (const kind of PREPARE_STEP_KINDS) {
|
|
223
|
+
if (!order.includes(kind))
|
|
224
|
+
order.push(kind);
|
|
217
225
|
}
|
|
218
|
-
|
|
219
|
-
|
|
226
|
+
return order;
|
|
227
|
+
}
|
|
228
|
+
/** Prepare steps for one option group (empty when the option was not given). */
|
|
229
|
+
function prepareStepsFor(kind, opts) {
|
|
230
|
+
switch (kind) {
|
|
231
|
+
case "rename": {
|
|
232
|
+
const renamings = (opts.rename ?? []).map((raw) => {
|
|
233
|
+
const [from, to,] = splitPair(raw, "--rename");
|
|
234
|
+
return { from, to, };
|
|
235
|
+
});
|
|
236
|
+
return renamings.length > 0 ? [processorStep("ColumnRenamer", { renamings, }),] : [];
|
|
237
|
+
}
|
|
238
|
+
case "fillEmpty":
|
|
239
|
+
return (opts.fillEmpty ?? []).map((raw) => {
|
|
240
|
+
const [column, value,] = splitPair(raw, "--fill-empty");
|
|
241
|
+
return processorStep("FillEmptyWithValue", {
|
|
242
|
+
appliesTo: "SINGLE_COLUMN",
|
|
243
|
+
columns: [column,],
|
|
244
|
+
value,
|
|
245
|
+
});
|
|
246
|
+
});
|
|
247
|
+
case "formula": {
|
|
248
|
+
if (!opts.formula)
|
|
249
|
+
return [];
|
|
250
|
+
const [column, expression,] = splitPair(opts.formula, "--formula");
|
|
251
|
+
return [processorStep("CreateColumnWithGREL", { column, expression, }),];
|
|
252
|
+
}
|
|
253
|
+
case "filter":
|
|
254
|
+
return opts.filter
|
|
255
|
+
? [processorStep("FilterOnCustomFormula", { expression: opts.filter, action: "KEEP_ROW", }),]
|
|
256
|
+
: [];
|
|
257
|
+
case "dropColumns":
|
|
258
|
+
return opts.dropColumns?.length
|
|
259
|
+
? [processorStep("ColumnsSelector", {
|
|
260
|
+
appliesTo: "COLUMNS",
|
|
261
|
+
columns: opts.dropColumns,
|
|
262
|
+
keep: false,
|
|
263
|
+
}),]
|
|
264
|
+
: [];
|
|
265
|
+
case "keepColumns":
|
|
266
|
+
return opts.keepColumns?.length
|
|
267
|
+
? [processorStep("ColumnsSelector", {
|
|
268
|
+
appliesTo: "COLUMNS",
|
|
269
|
+
columns: opts.keepColumns,
|
|
270
|
+
keep: true,
|
|
271
|
+
}),]
|
|
272
|
+
: [];
|
|
220
273
|
}
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
274
|
+
}
|
|
275
|
+
/**
|
|
276
|
+
* Prepare (shaker) steps from create options: rename, fill empty, formula
|
|
277
|
+
* column, row filter, drop columns, keep columns. They run in `stepOrder`
|
|
278
|
+
* (flag order on the CLI), defaulting to that sequence, so a step sees the
|
|
279
|
+
* columns the steps before it produced. Empty when no prepare option was given.
|
|
280
|
+
*/
|
|
281
|
+
export function prepareSteps(opts) {
|
|
282
|
+
return prepareStepOrder(opts).flatMap((kind) => prepareStepsFor(kind, opts));
|
|
283
|
+
}
|
|
284
|
+
/**
|
|
285
|
+
* Columns an expression refers to, conservatively: `val('name')` arguments
|
|
286
|
+
* plus bare identifiers outside string literals (not method names, calls, or
|
|
287
|
+
* members of another object).
|
|
288
|
+
*/
|
|
289
|
+
function expressionColumnReferences(expression) {
|
|
290
|
+
const references = new Set();
|
|
291
|
+
for (const match of expression.matchAll(/\bval\(\s*(['"])((?:\\.|(?!\1).)*)\1\s*\)/g)) {
|
|
292
|
+
references.add(match[2]);
|
|
227
293
|
}
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
columns: opts.keepColumns,
|
|
232
|
-
keep: true,
|
|
233
|
-
}));
|
|
294
|
+
const withoutStrings = expression.replace(/(['"])(?:\\.|(?!\1).)*\1/g, " ");
|
|
295
|
+
for (const match of withoutStrings.matchAll(/(?<![\w.$])([A-Za-z_]\w*)(?!\w)(?!\s*\()/g)) {
|
|
296
|
+
references.add(match[1]);
|
|
234
297
|
}
|
|
235
|
-
return
|
|
298
|
+
return references;
|
|
236
299
|
}
|
|
300
|
+
/**
|
|
301
|
+
* Warnings about the prepare options that DSS cannot report itself: a
|
|
302
|
+
* `--formula`/`--filter` that runs after a `--rename` but still names a
|
|
303
|
+
* column that rename removed (it would fail or read an empty column).
|
|
304
|
+
*/
|
|
305
|
+
export function recipeCreateWarnings(opts) {
|
|
306
|
+
const warnings = [];
|
|
307
|
+
const removed = new Map();
|
|
308
|
+
for (const kind of prepareStepOrder(opts)) {
|
|
309
|
+
if (kind === "rename") {
|
|
310
|
+
const renamings = (opts.rename ?? []).map((raw) => splitPair(raw, "--rename"));
|
|
311
|
+
const targets = new Set(renamings.map(([, to,]) => to));
|
|
312
|
+
for (const [from, to,] of renamings) {
|
|
313
|
+
if (!targets.has(from))
|
|
314
|
+
removed.set(from, to);
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
else if ((kind === "formula" && opts.formula) || (kind === "filter" && opts.filter)) {
|
|
318
|
+
const expression = kind === "formula"
|
|
319
|
+
? splitPair(opts.formula, "--formula")[1]
|
|
320
|
+
: opts.filter;
|
|
321
|
+
const stale = [...expressionColumnReferences(expression),].filter((name) => removed.has(name));
|
|
322
|
+
if (stale.length === 0)
|
|
323
|
+
continue;
|
|
324
|
+
const flag = kind === "formula" ? "--formula" : "--filter";
|
|
325
|
+
warnings.push({
|
|
326
|
+
code: "recipe_formula_uses_renamed_column",
|
|
327
|
+
flag,
|
|
328
|
+
columns: stale.map((name) => ({ renamed: name, to: removed.get(name), })),
|
|
329
|
+
message: `${flag} runs after --rename and refers to ${stale.map((name) => `"${name}"`).join(", ")}, which the rename removed.`,
|
|
330
|
+
hint: `Use the new name (${stale.map((name) => removed.get(name)).join(", ")}), or put ${flag} before --rename on the command line: steps run in flag order.`,
|
|
331
|
+
});
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
return warnings;
|
|
335
|
+
}
|
|
336
|
+
/**
|
|
337
|
+
* Recipe types created from `creationSettings.virtualInputs` (dataikuapi's
|
|
338
|
+
* VirtualInputsSingleOutputRecipeCreator subclasses: join, fuzzyjoin, geojoin,
|
|
339
|
+
* vstack, generate_features). Live DSS 15: without it these are created with
|
|
340
|
+
* no inputs (and `generate_features` fails with a NullPointerException).
|
|
341
|
+
*/
|
|
342
|
+
const VIRTUAL_INPUT_RECIPE_TYPES = {
|
|
343
|
+
vstack: true,
|
|
344
|
+
join: true,
|
|
345
|
+
fuzzyjoin: true,
|
|
346
|
+
geojoin: true,
|
|
347
|
+
generate_features: true,
|
|
348
|
+
};
|
|
237
349
|
/**
|
|
238
350
|
* Pure request construction for POST /recipes/ (no DSS calls). Shared with
|
|
239
351
|
* `recipe create --plan` so the plan equals the request the command sends.
|
|
@@ -301,6 +413,10 @@ export function buildRecipeCreateRequest(opts, pk) {
|
|
|
301
413
|
if (!Number.isFinite(fuzzyThreshold)) {
|
|
302
414
|
throw new ClientValidationError("fuzzyThreshold must be a finite number.", "validation_failed");
|
|
303
415
|
}
|
|
416
|
+
// Virtual-input recipes: DSS builds `inputs` from creationSettings.virtualInputs
|
|
417
|
+
// (dataikuapi VirtualInputsSingleOutputRecipeCreator) and ignores or drops the
|
|
418
|
+
// prototype's inputs, creating the recipe with `inputs: {}` and an empty payload.
|
|
419
|
+
const hasVirtualInputs = VIRTUAL_INPUT_RECIPE_TYPES[type] === true;
|
|
304
420
|
const recipePrototype = {
|
|
305
421
|
type,
|
|
306
422
|
name,
|
|
@@ -312,7 +428,7 @@ export function buildRecipeCreateRequest(opts, pk) {
|
|
|
312
428
|
if (payload !== undefined) {
|
|
313
429
|
creationSettings.script = payload;
|
|
314
430
|
}
|
|
315
|
-
if (
|
|
431
|
+
if (hasVirtualInputs) {
|
|
316
432
|
creationSettings.virtualInputs = recipeInputItems({ inputs, }).map((item) => item.ref);
|
|
317
433
|
}
|
|
318
434
|
return {
|
|
@@ -224,6 +224,18 @@ function shouldRetryRecipeCreateWithOutputProvisioning(error) {
|
|
|
224
224
|
// ---------------------------------------------------------------------------
|
|
225
225
|
// Resource
|
|
226
226
|
// ---------------------------------------------------------------------------
|
|
227
|
+
// DSS renames some recipe types server-side (e.g. prediction_scoring becomes
|
|
228
|
+
// "score_<inputDataset>") and documents the creation response as the final
|
|
229
|
+
// unique name: Body {"name": "recipe1"} ("Returns the final unique name of the
|
|
230
|
+
// recipe"). The response name is the contract — falling back to the requested
|
|
231
|
+
// name would mask a missing/invalid receipt, so a body without it is an error.
|
|
232
|
+
function requireRecipeName(created) {
|
|
233
|
+
const name = asString(created?.["name"]);
|
|
234
|
+
if (!name) {
|
|
235
|
+
throw unexpectedResponseError('DSS create-recipe response did not include the final recipe name (documented body: {"name": ...}).');
|
|
236
|
+
}
|
|
237
|
+
return name;
|
|
238
|
+
}
|
|
227
239
|
export class RecipesResource extends BaseResource {
|
|
228
240
|
/** List all recipes in a project. */
|
|
229
241
|
async list(projectKey) {
|
|
@@ -511,34 +523,74 @@ export class RecipesResource extends BaseResource {
|
|
|
511
523
|
}
|
|
512
524
|
};
|
|
513
525
|
let finalRecipeName;
|
|
514
|
-
//
|
|
515
|
-
//
|
|
516
|
-
//
|
|
517
|
-
//
|
|
518
|
-
//
|
|
519
|
-
//
|
|
526
|
+
// Datasets this call provisioned must not outlive a failed create: the
|
|
527
|
+
// failure is rethrown unchanged, with the rollback outcome in `details`.
|
|
528
|
+
// A failure can be ambiguous (timeout, 5xx after DSS committed, a body
|
|
529
|
+
// without the name), so a dataset some recipe already writes is kept:
|
|
530
|
+
// deleting it would break that recipe. DSS may rename the recipe, so the
|
|
531
|
+
// check reads the flow graph rather than the requested recipe name.
|
|
532
|
+
const rollbackProvisionedDatasets = async (error) => {
|
|
533
|
+
if (createdDatasets.length === 0)
|
|
534
|
+
throw error;
|
|
535
|
+
const deleted = [];
|
|
536
|
+
const failed = [];
|
|
537
|
+
const kept = [];
|
|
538
|
+
let producers;
|
|
539
|
+
try {
|
|
540
|
+
const { graph, } = await this.client.projects.flowTopology(pk);
|
|
541
|
+
producers = new Map(graph.edges.filter((edge) => edge.relation === "writes").map((edge) => [edge.to, edge.from,]));
|
|
542
|
+
}
|
|
543
|
+
catch (graphError) {
|
|
544
|
+
const reason = `could not read the flow graph to check for a recipe writing it (${graphError instanceof Error ? graphError.message : String(graphError)})`;
|
|
545
|
+
for (const name of createdDatasets)
|
|
546
|
+
kept.push({ name, reason, });
|
|
547
|
+
}
|
|
548
|
+
for (const name of createdDatasets) {
|
|
549
|
+
if (!producers)
|
|
550
|
+
break;
|
|
551
|
+
const producer = producers.get(name);
|
|
552
|
+
if (producer !== undefined) {
|
|
553
|
+
kept.push({ name, reason: `written by recipe ${producer}`, });
|
|
554
|
+
continue;
|
|
555
|
+
}
|
|
556
|
+
try {
|
|
557
|
+
await this.client.del(`/public/api/projects/${enc}/datasets/${encodeURIComponent(name)}`);
|
|
558
|
+
deleted.push(name);
|
|
559
|
+
}
|
|
560
|
+
catch (deleteError) {
|
|
561
|
+
failed.push({
|
|
562
|
+
name,
|
|
563
|
+
error: deleteError instanceof Error ? deleteError.message : String(deleteError),
|
|
564
|
+
});
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
if (error instanceof DataikuError || error instanceof ClientValidationError) {
|
|
568
|
+
error.details = {
|
|
569
|
+
...error.details,
|
|
570
|
+
createdDatasetsDeleted: deleted,
|
|
571
|
+
createdDatasetsNotDeleted: failed,
|
|
572
|
+
...(kept.length > 0 ? { createdDatasetsKept: kept, } : {}),
|
|
573
|
+
};
|
|
574
|
+
}
|
|
575
|
+
throw error;
|
|
576
|
+
};
|
|
520
577
|
try {
|
|
521
|
-
|
|
578
|
+
try {
|
|
579
|
+
if (rawConnection)
|
|
580
|
+
await provisionOutputDatasets();
|
|
581
|
+
finalRecipeName = requireRecipeName(await createRecipe());
|
|
582
|
+
}
|
|
583
|
+
catch (error) {
|
|
584
|
+
if (!shouldRetryRecipeCreateWithOutputProvisioning(error)) {
|
|
585
|
+
throw error;
|
|
586
|
+
}
|
|
587
|
+
usedOutputProvisioningFallback = true;
|
|
522
588
|
await provisionOutputDatasets();
|
|
523
|
-
|
|
524
|
-
const receivedName = asString(created?.["name"]);
|
|
525
|
-
if (!receivedName) {
|
|
526
|
-
throw unexpectedResponseError('DSS create-recipe response did not include the final recipe name (documented body: {"name": ...}).');
|
|
589
|
+
finalRecipeName = requireRecipeName(await createRecipe());
|
|
527
590
|
}
|
|
528
|
-
finalRecipeName = receivedName;
|
|
529
591
|
}
|
|
530
592
|
catch (error) {
|
|
531
|
-
|
|
532
|
-
throw error;
|
|
533
|
-
}
|
|
534
|
-
usedOutputProvisioningFallback = true;
|
|
535
|
-
await provisionOutputDatasets();
|
|
536
|
-
const retryCreated = await createRecipe();
|
|
537
|
-
const retryName = asString(retryCreated?.["name"]);
|
|
538
|
-
if (!retryName) {
|
|
539
|
-
throw unexpectedResponseError('DSS create-recipe response did not include the final recipe name (documented body: {"name": ...}).');
|
|
540
|
-
}
|
|
541
|
-
finalRecipeName = retryName;
|
|
593
|
+
await rollbackProvisionedDatasets(error);
|
|
542
594
|
}
|
|
543
595
|
// For join recipes: configure join conditions after creation
|
|
544
596
|
let joinConfigured = false;
|
|
@@ -684,6 +736,7 @@ export class RecipesResource extends BaseResource {
|
|
|
684
736
|
}
|
|
685
737
|
}
|
|
686
738
|
return {
|
|
739
|
+
...(opts.name && opts.name !== finalRecipeName ? { requestedName: opts.name, } : {}),
|
|
687
740
|
recipeName: finalRecipeName,
|
|
688
741
|
type,
|
|
689
742
|
createdDatasets,
|
|
@@ -76,31 +76,70 @@ function sqlErrorDetailFromBody(body) {
|
|
|
76
76
|
const match = trimmed.match(/\b[A-Z_]+(?:_ERROR|_NOT_FOUND|_MISMATCH|_DENIED|_EXCEEDED)\b[:\s-]+[^\n\r]+/);
|
|
77
77
|
return match?.[0] ?? (isLikelySqlErrorDetail(trimmed) ? trimmed.slice(0, 500) : undefined);
|
|
78
78
|
}
|
|
79
|
+
/**
|
|
80
|
+
* JDBC/driver failures that mean the connection to the database broke or timed
|
|
81
|
+
* out mid-statement: a DML statement may already have committed, so these stay
|
|
82
|
+
* an unknown outcome even though they are SQL/JDBC-flavored.
|
|
83
|
+
*/
|
|
84
|
+
const SQL_CONNECTION_LOSS_TYPE = /transient|recoverable|timeout/i;
|
|
85
|
+
const SQL_CONNECTION_LOSS_TEXT = /i\/o error|communication (?:link|error|failure)|connection (?:reset|closed|refused|lost|abort)|broken pipe|socket|timed out|network/i;
|
|
86
|
+
/**
|
|
87
|
+
* Whether DSS answered with a definite refusal of the statement, as opposed to
|
|
88
|
+
* an unknown outcome (transport reset, timeout, bare gateway error). DSS runs the
|
|
89
|
+
* statement before answering the start POST, so a SQL-flavored error body (JDBC
|
|
90
|
+
* error, or an error type from a SQL/JDBC class) or a body DSS itself classifies
|
|
91
|
+
* as permission/not-found means the database rejected the statement — unless
|
|
92
|
+
* the error reports a lost or timed-out database connection.
|
|
93
|
+
*/
|
|
94
|
+
function isDefiniteSqlRejection(error, sqlDetail) {
|
|
95
|
+
if (error.status < 400)
|
|
96
|
+
return false;
|
|
97
|
+
let errorType;
|
|
98
|
+
try {
|
|
99
|
+
errorType = asRecord(JSON.parse(error.body))?.errorType;
|
|
100
|
+
}
|
|
101
|
+
catch {
|
|
102
|
+
errorType = undefined;
|
|
103
|
+
}
|
|
104
|
+
if (typeof errorType === "string" && SQL_CONNECTION_LOSS_TYPE.test(errorType))
|
|
105
|
+
return false;
|
|
106
|
+
if (SQL_CONNECTION_LOSS_TEXT.test(error.body))
|
|
107
|
+
return false;
|
|
108
|
+
if (sqlDetail)
|
|
109
|
+
return true;
|
|
110
|
+
if (error.category === "forbidden" || error.category === "not_found")
|
|
111
|
+
return true;
|
|
112
|
+
return typeof errorType === "string" && /sql|jdbc/i.test(errorType);
|
|
113
|
+
}
|
|
79
114
|
function withSqlErrorContext(error) {
|
|
80
115
|
if (error instanceof DataikuError) {
|
|
81
116
|
const detail = sqlErrorDetailFromBody(error.body);
|
|
82
|
-
|
|
117
|
+
const definiteFailure = isDefiniteSqlRejection(error, detail);
|
|
118
|
+
if (detail || definiteFailure) {
|
|
83
119
|
let body = error.body;
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
120
|
+
if (detail) {
|
|
121
|
+
try {
|
|
122
|
+
const parsed = JSON.parse(error.body);
|
|
123
|
+
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
124
|
+
body = JSON.stringify({
|
|
125
|
+
...parsed,
|
|
126
|
+
message: `SQL query failed: ${detail}`,
|
|
127
|
+
sqlError: detail,
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
else {
|
|
131
|
+
body = `SQL query failed: ${detail}\n${error.body}`;
|
|
132
|
+
}
|
|
92
133
|
}
|
|
93
|
-
|
|
134
|
+
catch {
|
|
94
135
|
body = `SQL query failed: ${detail}\n${error.body}`;
|
|
95
136
|
}
|
|
96
137
|
}
|
|
97
|
-
catch {
|
|
98
|
-
body = `SQL query failed: ${detail}\n${error.body}`;
|
|
99
|
-
}
|
|
100
138
|
throw new DataikuError(error.status, error.statusText, body, error.retry, error.requestId, {
|
|
101
139
|
target: error.trustedTarget,
|
|
102
140
|
elapsedMs: error.trustedElapsedMs,
|
|
103
141
|
bodyTruncated: error.bodyTruncated,
|
|
142
|
+
definiteFailure,
|
|
104
143
|
});
|
|
105
144
|
}
|
|
106
145
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { ClientValidationError } from "../errors.js";
|
|
2
|
+
import type { WikiArticleAttachment, WikiArticleData, WikiSettings } from "../schemas.js";
|
|
2
3
|
import { BaseResource } from "./base.js";
|
|
3
4
|
export interface WikiArticleCreateOptions {
|
|
4
5
|
name: string;
|
|
@@ -12,6 +13,37 @@ export interface WikiArticleUpdateOptions {
|
|
|
12
13
|
data?: Record<string, unknown>;
|
|
13
14
|
projectKey?: string;
|
|
14
15
|
}
|
|
16
|
+
export interface WikiAttachResult {
|
|
17
|
+
articleId: string;
|
|
18
|
+
projectKey: string;
|
|
19
|
+
fileName: string;
|
|
20
|
+
bytes: number;
|
|
21
|
+
/** The attachment DSS created, as re-read from the article. */
|
|
22
|
+
attachment: WikiArticleAttachment;
|
|
23
|
+
/** All attachments of the article after the upload. */
|
|
24
|
+
attachments: WikiArticleAttachment[];
|
|
25
|
+
}
|
|
26
|
+
export interface WikiDetachResult {
|
|
27
|
+
articleId: string;
|
|
28
|
+
projectKey: string;
|
|
29
|
+
/** The attachment that was removed. */
|
|
30
|
+
detached: WikiArticleAttachment;
|
|
31
|
+
/** Attachments left on the article. */
|
|
32
|
+
attachments: WikiArticleAttachment[];
|
|
33
|
+
}
|
|
34
|
+
export interface LocalWikiAttachmentFile {
|
|
35
|
+
path: string;
|
|
36
|
+
fileName: string;
|
|
37
|
+
bytes: number;
|
|
38
|
+
}
|
|
39
|
+
/** Attachments of an article; DSS omits the field when there are none. */
|
|
40
|
+
export declare function wikiAttachments(data: WikiArticleData): WikiArticleAttachment[];
|
|
41
|
+
/** The attachment with exactly this smart id, or undefined. */
|
|
42
|
+
export declare function findWikiAttachment(data: WikiArticleData, smartId: string): WikiArticleAttachment | undefined;
|
|
43
|
+
/** Coded not_found error for a smart id that is not attached to the article. */
|
|
44
|
+
export declare function wikiAttachmentNotFound(data: WikiArticleData, smartId: string): ClientValidationError;
|
|
45
|
+
/** Check that a local path is a readable regular file before any DSS call. */
|
|
46
|
+
export declare function statWikiAttachmentFile(localPath: string): Promise<LocalWikiAttachmentFile>;
|
|
15
47
|
export declare class WikiResource extends BaseResource {
|
|
16
48
|
settings(projectKey?: string): Promise<WikiSettings>;
|
|
17
49
|
/**
|
|
@@ -24,4 +56,18 @@ export declare class WikiResource extends BaseResource {
|
|
|
24
56
|
create(opts: WikiArticleCreateOptions): Promise<WikiArticleData>;
|
|
25
57
|
update(articleIdOrName: string, opts: WikiArticleUpdateOptions): Promise<WikiArticleData>;
|
|
26
58
|
delete(articleIdOrName: string, projectKey?: string): Promise<void>;
|
|
59
|
+
/**
|
|
60
|
+
* Upload a local file and attach it to an article (multipart POST to
|
|
61
|
+
* /wiki/{id}/upload). DSS names the attachment after the file's basename and
|
|
62
|
+
* types it from the extension. The article is re-read to identify the new
|
|
63
|
+
* attachment.
|
|
64
|
+
*/
|
|
65
|
+
attach(articleIdOrName: string, localPath: string, projectKey?: string): Promise<WikiAttachResult>;
|
|
66
|
+
/**
|
|
67
|
+
* Remove exactly one attachment (by smart id) from an article: the article is
|
|
68
|
+
* read, the entry filtered out of `article.attachments`, and the whole
|
|
69
|
+
* article PUT back. Throws a coded `not_found` error when the smart id is not
|
|
70
|
+
* attached.
|
|
71
|
+
*/
|
|
72
|
+
detach(articleIdOrName: string, smartId: string, projectKey?: string): Promise<WikiDetachResult>;
|
|
27
73
|
}
|
|
@@ -1,6 +1,36 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { ClientValidationError, } from "../errors.js";
|
|
1
4
|
import { WikiArticleDataArraySchema, WikiArticleDataSchema, WikiSettingsSchema, } from "../schemas.js";
|
|
2
5
|
import { deepMerge, } from "../utils/deep-merge.js";
|
|
3
6
|
import { BaseResource, } from "./base.js";
|
|
7
|
+
/** Attachments of an article; DSS omits the field when there are none. */
|
|
8
|
+
export function wikiAttachments(data) {
|
|
9
|
+
return data.article.attachments ?? [];
|
|
10
|
+
}
|
|
11
|
+
/** The attachment with exactly this smart id, or undefined. */
|
|
12
|
+
export function findWikiAttachment(data, smartId) {
|
|
13
|
+
return wikiAttachments(data).find((attachment) => attachment.smartId === smartId);
|
|
14
|
+
}
|
|
15
|
+
/** Coded not_found error for a smart id that is not attached to the article. */
|
|
16
|
+
export function wikiAttachmentNotFound(data, smartId) {
|
|
17
|
+
const available = wikiAttachments(data).flatMap((attachment) => attachment.smartId === undefined ? [] : [attachment.smartId,]);
|
|
18
|
+
return new ClientValidationError(`Attachment ${smartId} not found on wiki article ${data.article.id}.`, "not_found", "List attachments with `dss wiki get ARTICLE_ID` (article.attachments[].smartId), or pass --if-exists to treat absence as success.", { articleId: data.article.id, smartId, availableSmartIds: available, });
|
|
19
|
+
}
|
|
20
|
+
/** Check that a local path is a readable regular file before any DSS call. */
|
|
21
|
+
export async function statWikiAttachmentFile(localPath) {
|
|
22
|
+
let stat;
|
|
23
|
+
try {
|
|
24
|
+
stat = await fs.promises.stat(localPath);
|
|
25
|
+
}
|
|
26
|
+
catch (error) {
|
|
27
|
+
throw new ClientValidationError(`Could not read attachment file ${localPath}: ${error instanceof Error ? error.message : String(error)}.`, "not_found", "Verify the file path and that it is readable.");
|
|
28
|
+
}
|
|
29
|
+
if (!stat.isFile()) {
|
|
30
|
+
throw new ClientValidationError(`${localPath} is not a regular file.`);
|
|
31
|
+
}
|
|
32
|
+
return { path: localPath, fileName: path.basename(localPath), bytes: stat.size, };
|
|
33
|
+
}
|
|
4
34
|
const WIKI_LIST_CONCURRENCY = 4;
|
|
5
35
|
function taxonomyIds(nodes) {
|
|
6
36
|
const ids = [];
|
|
@@ -77,4 +107,63 @@ export class WikiResource extends BaseResource {
|
|
|
77
107
|
const current = await this.get(articleIdOrName, projectKey);
|
|
78
108
|
await this.client.del(`/public/api/projects/${this.enc(projectKey)}/wiki/${encodeURIComponent(current.article.id)}`);
|
|
79
109
|
}
|
|
110
|
+
/**
|
|
111
|
+
* Upload a local file and attach it to an article (multipart POST to
|
|
112
|
+
* /wiki/{id}/upload). DSS names the attachment after the file's basename and
|
|
113
|
+
* types it from the extension. The article is re-read to identify the new
|
|
114
|
+
* attachment.
|
|
115
|
+
*/
|
|
116
|
+
async attach(articleIdOrName, localPath, projectKey) {
|
|
117
|
+
const file = await statWikiAttachmentFile(localPath);
|
|
118
|
+
const pk = this.resolveProjectKey(projectKey);
|
|
119
|
+
const before = await this.get(articleIdOrName, pk);
|
|
120
|
+
const articleId = before.article.id;
|
|
121
|
+
await this.client.uploadJson(`/public/api/projects/${encodeURIComponent(pk)}/wiki/${encodeURIComponent(articleId)}/upload`, localPath, file.fileName);
|
|
122
|
+
const after = await this.get(articleId, pk);
|
|
123
|
+
const knownIds = new Set(wikiAttachments(before).map((attachment) => attachment.smartId));
|
|
124
|
+
let created = wikiAttachments(after).filter((attachment) => !knownIds.has(attachment.smartId));
|
|
125
|
+
if (created.length > 1) {
|
|
126
|
+
created = created.filter((attachment) => attachment.details?.["objectDisplayName"] === file.fileName
|
|
127
|
+
&& attachment.details?.["size"] === file.bytes);
|
|
128
|
+
}
|
|
129
|
+
const attachment = created[0];
|
|
130
|
+
if (created.length !== 1 || attachment === undefined || attachment.smartId === undefined) {
|
|
131
|
+
throw new ClientValidationError(`DSS did not verify upload of ${file.fileName} to wiki article ${articleId}.`, "ambiguous_outcome", "The upload returned successfully, but the article did not show exactly one new attachment. Inspect the article (dss wiki get) before retrying.", { articleId, expectedBytes: file.bytes, newAttachments: created, });
|
|
132
|
+
}
|
|
133
|
+
return {
|
|
134
|
+
articleId,
|
|
135
|
+
projectKey: pk,
|
|
136
|
+
fileName: file.fileName,
|
|
137
|
+
bytes: file.bytes,
|
|
138
|
+
attachment,
|
|
139
|
+
attachments: wikiAttachments(after),
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Remove exactly one attachment (by smart id) from an article: the article is
|
|
144
|
+
* read, the entry filtered out of `article.attachments`, and the whole
|
|
145
|
+
* article PUT back. Throws a coded `not_found` error when the smart id is not
|
|
146
|
+
* attached.
|
|
147
|
+
*/
|
|
148
|
+
async detach(articleIdOrName, smartId, projectKey) {
|
|
149
|
+
const pk = this.resolveProjectKey(projectKey);
|
|
150
|
+
const current = await this.get(articleIdOrName, pk);
|
|
151
|
+
const detached = findWikiAttachment(current, smartId);
|
|
152
|
+
if (detached === undefined)
|
|
153
|
+
throw wikiAttachmentNotFound(current, smartId);
|
|
154
|
+
const remaining = wikiAttachments(current).filter((attachment) => attachment.smartId !== smartId);
|
|
155
|
+
const raw = await this.client.put(`/public/api/projects/${encodeURIComponent(pk)}/wiki/${encodeURIComponent(current.article.id)}`, { ...current, article: { ...current.article, attachments: remaining, }, });
|
|
156
|
+
const saved = this.client.safeParse(WikiArticleDataSchema, raw, "wiki.detach");
|
|
157
|
+
const after = wikiAttachments(saved);
|
|
158
|
+
if (findWikiAttachment(saved, smartId) !== undefined
|
|
159
|
+
|| after.length !== remaining.length) {
|
|
160
|
+
throw new ClientValidationError(`DSS did not verify removal of attachment ${smartId} from wiki article ${current.article.id}.`, "ambiguous_outcome", "The save returned successfully, but the article's attachments are not the expected remainder. Inspect the article (dss wiki get) before retrying.", {
|
|
161
|
+
articleId: current.article.id,
|
|
162
|
+
smartId,
|
|
163
|
+
expectedRemaining: remaining.length,
|
|
164
|
+
actualRemaining: after.length,
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
return { articleId: current.article.id, projectKey: pk, detached, attachments: after, };
|
|
168
|
+
}
|
|
80
169
|
}
|
|
@@ -345,6 +345,10 @@ function renderMermaid(map) {
|
|
|
345
345
|
lines.push(` ${idByNode.get(nodeId)}["${mermaidLabel(readableNode(node))}"]`);
|
|
346
346
|
}
|
|
347
347
|
lines.push(" end");
|
|
348
|
+
// Zone colors come from DSS; only plain #RRGGBB values reach the style line.
|
|
349
|
+
if (zone.color && /^#[0-9a-fA-F]{6}$/.test(zone.color)) {
|
|
350
|
+
lines.push(` style z${zoneIndex} fill:${zone.color}22,stroke:${zone.color}`);
|
|
351
|
+
}
|
|
348
352
|
}
|
|
349
353
|
for (const edge of map.edges) {
|
|
350
354
|
const from = idByNode.get(edge.from);
|
|
@@ -7,8 +7,10 @@
|
|
|
7
7
|
* - optionally redacts userinfo embedded in HTTP(S) URL authorities
|
|
8
8
|
* (`https://user:pass@host/...` -> `https://[redacted]@host/...`).
|
|
9
9
|
*
|
|
10
|
-
* The SDK resource layer forwards values verbatim to DSS over TLS;
|
|
11
|
-
* is a CLI presentation concern and must never leak secret
|
|
10
|
+
* The SDK resource layer forwards values verbatim to DSS over TLS; sanitizing
|
|
11
|
+
* structured output is a CLI presentation concern and must never leak secret
|
|
12
|
+
* material instead. DSS-produced read text that consumers never need secrets
|
|
13
|
+
* from (job logs) is scrubbed at the SDK download via `redactCredentialPairs`.
|
|
12
14
|
*/
|
|
13
15
|
/** Marker replacing every redacted value or secret occurrence. */
|
|
14
16
|
export declare const SECRET_REDACTED = "[redacted]";
|
|
@@ -46,6 +48,13 @@ export declare function redactUrlUserinfo(text: string): string;
|
|
|
46
48
|
* recurse, arrays recurse. Non-string scalars pass through unchanged.
|
|
47
49
|
*/
|
|
48
50
|
export declare function sanitizeSecrets<T>(value: T, options: SecretSanitizeOptions): T;
|
|
51
|
+
/**
|
|
52
|
+
* Redact the string value of every JSON-style `"key":"value"` pair in free
|
|
53
|
+
* text whose key is credential-like (e.g. the `jobTicketSecret` DSS writes
|
|
54
|
+
* into job logs). Bare `key` fields are not redacted here: in log text they
|
|
55
|
+
* name ordinary identifiers far more often than secrets.
|
|
56
|
+
*/
|
|
57
|
+
export declare function redactCredentialPairs(text: string): string;
|
|
49
58
|
/**
|
|
50
59
|
* Every string value under a credential-like key (any depth), plus bare
|
|
51
60
|
* `key` fields such as `authRealm.key`. Collected from a command's own JSON
|