dataiku-sdk 3.8.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -5
- package/dist/build-metadata.json +1 -1
- package/dist/packages/types/src/index.d.ts +4 -0
- package/dist/packages/types/src/index.js +4 -0
- package/dist/src/cli/coerce.d.ts +6 -0
- package/dist/src/cli/coerce.js +9 -0
- package/dist/src/cli/command-syntax.json +36 -0
- package/dist/src/cli/commands/app.js +2 -2
- package/dist/src/cli/commands/code-env.js +5 -6
- package/dist/src/cli/commands/code.js +16 -2
- package/dist/src/cli/commands/dashboard.js +1 -2
- package/dist/src/cli/commands/dataset.js +2 -2
- package/dist/src/cli/commands/flow-zone.js +1 -1
- package/dist/src/cli/commands/job.js +101 -22
- package/dist/src/cli/commands/plugin.js +7 -7
- package/dist/src/cli/commands/project-library.js +2 -2
- package/dist/src/cli/commands/project.js +25 -7
- package/dist/src/cli/commands/recipe.js +11 -19
- package/dist/src/cli/commands/scenario.js +2 -4
- package/dist/src/cli/contract.js +29 -10
- package/dist/src/cli/env.js +2 -3
- package/dist/src/cli/helpers/app-permissions.d.ts +1 -1
- package/dist/src/cli/helpers/app-permissions.js +2 -2
- package/dist/src/cli/helpers/cleanup.d.ts +2 -0
- package/dist/src/cli/helpers/cleanup.js +122 -8
- package/dist/src/cli/helpers/flow-zone.d.ts +1 -1
- package/dist/src/cli/helpers/flow-zone.js +3 -4
- package/dist/src/cli/list-output.d.ts +5 -0
- package/dist/src/cli/list-output.js +11 -2
- package/dist/src/cli/list-shapes.d.ts +6 -0
- package/dist/src/cli/list-shapes.js +7 -1
- package/dist/src/cli/output-schemas.d.ts +1 -1
- package/dist/src/cli/output-schemas.js +28 -4
- package/dist/src/cli/output.js +7 -0
- package/dist/src/cli/project-map-output.d.ts +24 -0
- package/dist/src/cli/project-map-output.js +21 -0
- package/dist/src/cli/read-shapes.d.ts +14 -0
- package/dist/src/cli/read-shapes.js +114 -0
- package/dist/src/cli/result-shapes.d.ts +6 -0
- package/dist/src/cli/result-shapes.js +235 -0
- package/dist/src/cli/version.d.ts +2 -2
- package/dist/src/cli/version.js +4 -5
- package/dist/src/cli.js +77 -33
- package/dist/src/client.d.ts +14 -2
- package/dist/src/client.js +18 -10
- package/dist/src/generated/action-output-schemas.json +242 -35
- package/dist/src/index.d.ts +1 -1
- package/dist/src/resources/datasets.d.ts +11 -1
- package/dist/src/resources/datasets.js +91 -30
- package/dist/src/resources/jobs.d.ts +37 -6
- package/dist/src/resources/jobs.js +110 -44
- package/dist/src/resources/projects.js +2 -2
- package/dist/src/resources/recipes.d.ts +12 -1
- package/dist/src/resources/recipes.js +39 -12
- package/dist/src/resources/scenarios.d.ts +8 -1
- package/dist/src/resources/scenarios.js +51 -12
- package/dist/src/resources/wiki.js +3 -7
- package/dist/src/skill.js +1 -2
- package/dist/src/utils/project-archive.d.ts +25 -0
- package/dist/src/utils/project-archive.js +216 -5
- package/dist/src/utils/response-file.js +15 -8
- package/dist/src/utils/secret-sanitize.d.ts +11 -2
- package/dist/src/utils/secret-sanitize.js +23 -6
- package/package.json +13 -14
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +1 -1
- package/skills/dataiku-dss/references/discovery.md +2 -2
- package/skills/dataiku-dss/references/troubleshooting.md +2 -2
|
@@ -35,6 +35,25 @@ export interface ProjectArchiveIssue {
|
|
|
35
35
|
message: string;
|
|
36
36
|
member?: string;
|
|
37
37
|
}
|
|
38
|
+
/**
|
|
39
|
+
* One credential-bearing structure found in an archive member. Carries the
|
|
40
|
+
* member path and the JSON key path / field name only, never the value.
|
|
41
|
+
* - `member_name`: the member itself is a credential store (for example
|
|
42
|
+
* `project_config/apikeys.json`) and is not empty.
|
|
43
|
+
* - `json_key`: a credential-named JSON key holds a non-empty string value
|
|
44
|
+
* (or a container of them), e.g. `$.standard.DHL_API_KEY` or a nested
|
|
45
|
+
* web-app `apiKey`.
|
|
46
|
+
* - `list_variable`: a list-form `{name, value}` entry whose `name` is
|
|
47
|
+
* credential-named, e.g. `{name: "X_API_KEY", value: "..."}`.
|
|
48
|
+
*/
|
|
49
|
+
export interface ProjectArchiveCredentialFinding {
|
|
50
|
+
member: string;
|
|
51
|
+
rule: "member_name" | "json_key" | "list_variable";
|
|
52
|
+
/** JSON path of the credential value inside the member (absent for `member_name`). */
|
|
53
|
+
keyPath?: string;
|
|
54
|
+
/** Credential-named key or list-form variable name (absent for `member_name`). */
|
|
55
|
+
field?: string;
|
|
56
|
+
}
|
|
38
57
|
export interface ProjectArchiveInspection {
|
|
39
58
|
filePath: string;
|
|
40
59
|
sizeBytes: number;
|
|
@@ -42,6 +61,12 @@ export interface ProjectArchiveInspection {
|
|
|
42
61
|
sourceProjectKey?: string;
|
|
43
62
|
valid: boolean;
|
|
44
63
|
issues: ProjectArchiveIssue[];
|
|
64
|
+
/**
|
|
65
|
+
* Credential inventory: member paths and key paths/field names of
|
|
66
|
+
* credential-bearing structures. Values are never included. Each finding
|
|
67
|
+
* is also summarized by a `credential_structure` warning issue.
|
|
68
|
+
*/
|
|
69
|
+
credentialFindings: ProjectArchiveCredentialFinding[];
|
|
45
70
|
}
|
|
46
71
|
/**
|
|
47
72
|
* Inspect a DSS project export ZIP archive without extracting anything to
|
|
@@ -345,6 +345,190 @@ function residueIssue(member, counts) {
|
|
|
345
345
|
member,
|
|
346
346
|
};
|
|
347
347
|
}
|
|
348
|
+
/** Cap on credential findings recorded per member, bounding inventory size. */
|
|
349
|
+
const MAX_CREDENTIAL_FINDINGS_PER_MEMBER = 200;
|
|
350
|
+
/** Deepest JSON nesting the credential walker follows. */
|
|
351
|
+
const MAX_CREDENTIAL_WALK_DEPTH = 32;
|
|
352
|
+
const PROJECT_CONFIG_ROOT = "project_config";
|
|
353
|
+
/** Credential-store member names under `project_config/` (api keys, secrets, credentials files). */
|
|
354
|
+
const CREDENTIAL_MEMBER_RE = /^(?:api[-_]?keys?|secrets?|credentials?)\.json$/i;
|
|
355
|
+
const CREDENTIAL_SINGLE_TOKENS = {
|
|
356
|
+
secret: true,
|
|
357
|
+
secrets: true,
|
|
358
|
+
token: true,
|
|
359
|
+
tokens: true,
|
|
360
|
+
password: true,
|
|
361
|
+
passwords: true,
|
|
362
|
+
passwd: true,
|
|
363
|
+
passphrase: true,
|
|
364
|
+
credential: true,
|
|
365
|
+
credentials: true,
|
|
366
|
+
apikey: true,
|
|
367
|
+
apikeys: true,
|
|
368
|
+
auth: true,
|
|
369
|
+
oauth: true,
|
|
370
|
+
authorization: true,
|
|
371
|
+
};
|
|
372
|
+
/**
|
|
373
|
+
* Trailing key words that describe a credential rather than hold one
|
|
374
|
+
* (`tokenType`, `authMode`, `secretName`, `passwordColumn`, ...).
|
|
375
|
+
*/
|
|
376
|
+
const NON_SECRET_SUFFIX_TOKENS = {
|
|
377
|
+
type: true,
|
|
378
|
+
mode: true,
|
|
379
|
+
method: true,
|
|
380
|
+
kind: true,
|
|
381
|
+
provider: true,
|
|
382
|
+
column: true,
|
|
383
|
+
columns: true,
|
|
384
|
+
field: true,
|
|
385
|
+
fields: true,
|
|
386
|
+
name: true,
|
|
387
|
+
names: true,
|
|
388
|
+
label: true,
|
|
389
|
+
description: true,
|
|
390
|
+
url: true,
|
|
391
|
+
uri: true,
|
|
392
|
+
path: true,
|
|
393
|
+
scope: true,
|
|
394
|
+
scopes: true,
|
|
395
|
+
format: true,
|
|
396
|
+
policy: true,
|
|
397
|
+
enabled: true,
|
|
398
|
+
required: true,
|
|
399
|
+
};
|
|
400
|
+
/** Split `DHL_API_KEY`, `apiKey`, `x-api-key` and `APIKey` into lower-case words. */
|
|
401
|
+
function keyTokens(key) {
|
|
402
|
+
return key
|
|
403
|
+
.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
|
|
404
|
+
.replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
|
|
405
|
+
.split(/[^A-Za-z0-9]+/)
|
|
406
|
+
.filter((token) => token.length > 0)
|
|
407
|
+
.map((token) => token.toLowerCase());
|
|
408
|
+
}
|
|
409
|
+
function isCredentialName(key) {
|
|
410
|
+
const tokens = keyTokens(key);
|
|
411
|
+
const last = tokens.at(-1);
|
|
412
|
+
if (last !== undefined && NON_SECRET_SUFFIX_TOKENS[last] === true)
|
|
413
|
+
return false;
|
|
414
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
415
|
+
const token = tokens[i];
|
|
416
|
+
if (CREDENTIAL_SINGLE_TOKENS[token] === true)
|
|
417
|
+
return true;
|
|
418
|
+
const next = tokens[i + 1];
|
|
419
|
+
if ((token === "api" || token === "private") && (next === "key" || next === "keys"))
|
|
420
|
+
return true;
|
|
421
|
+
}
|
|
422
|
+
return false;
|
|
423
|
+
}
|
|
424
|
+
/** Non-empty string that is not a pure `${VARIABLE}` reference. */
|
|
425
|
+
function isLiteralSecretValue(value) {
|
|
426
|
+
if (typeof value !== "string")
|
|
427
|
+
return false;
|
|
428
|
+
const trimmed = value.trim();
|
|
429
|
+
return trimmed.length > 0 && !/^\$\{[^}]*\}$/.test(trimmed);
|
|
430
|
+
}
|
|
431
|
+
/** True for null, empty strings and containers holding only such values. */
|
|
432
|
+
function isEmptyJsonValue(value) {
|
|
433
|
+
if (value === null || value === undefined)
|
|
434
|
+
return true;
|
|
435
|
+
if (typeof value === "string")
|
|
436
|
+
return value.trim().length === 0;
|
|
437
|
+
if (Array.isArray(value))
|
|
438
|
+
return value.every(isEmptyJsonValue);
|
|
439
|
+
if (typeof value === "object")
|
|
440
|
+
return Object.values(value).every(isEmptyJsonValue);
|
|
441
|
+
return false;
|
|
442
|
+
}
|
|
443
|
+
function jsonPathSegment(key) {
|
|
444
|
+
const safe = key.length > 80 ? `${key.slice(0, 80)}...` : key;
|
|
445
|
+
return /^[A-Za-z_$][\w$]*$/.test(safe) ? `.${safe}` : `[${JSON.stringify(safe)}]`;
|
|
446
|
+
}
|
|
447
|
+
/**
|
|
448
|
+
* Structural credential detection for one `project_config/` member: the
|
|
449
|
+
* member-name rule plus a JSON-aware key walk. Only member paths and key
|
|
450
|
+
* paths/field names are returned, never values.
|
|
451
|
+
*/
|
|
452
|
+
function collectCredentialFindings(member, text) {
|
|
453
|
+
const findings = [];
|
|
454
|
+
const full = () => findings.length >= MAX_CREDENTIAL_FINDINGS_PER_MEMBER;
|
|
455
|
+
let parsed;
|
|
456
|
+
let isJson = false;
|
|
457
|
+
try {
|
|
458
|
+
parsed = JSON.parse(text);
|
|
459
|
+
isJson = true;
|
|
460
|
+
}
|
|
461
|
+
catch {
|
|
462
|
+
// Not JSON: only the member-name rule can apply.
|
|
463
|
+
}
|
|
464
|
+
const basename = member.slice(member.lastIndexOf("/") + 1);
|
|
465
|
+
if (CREDENTIAL_MEMBER_RE.test(basename)) {
|
|
466
|
+
const empty = isJson ? isEmptyJsonValue(parsed) : text.trim().length === 0;
|
|
467
|
+
if (!empty)
|
|
468
|
+
findings.push({ member, rule: "member_name", });
|
|
469
|
+
}
|
|
470
|
+
if (!isJson)
|
|
471
|
+
return findings;
|
|
472
|
+
const visit = (node, path, inCredential, field, depth) => {
|
|
473
|
+
if (full() || depth > MAX_CREDENTIAL_WALK_DEPTH)
|
|
474
|
+
return;
|
|
475
|
+
if (typeof node === "string") {
|
|
476
|
+
if (inCredential && isLiteralSecretValue(node)) {
|
|
477
|
+
findings.push({ member, rule: "json_key", keyPath: path, field, });
|
|
478
|
+
}
|
|
479
|
+
return;
|
|
480
|
+
}
|
|
481
|
+
if (Array.isArray(node)) {
|
|
482
|
+
node.forEach((item, index) => visit(item, `${path}[${index}]`, inCredential, field, depth + 1));
|
|
483
|
+
return;
|
|
484
|
+
}
|
|
485
|
+
if (node === null || typeof node !== "object")
|
|
486
|
+
return;
|
|
487
|
+
const record = node;
|
|
488
|
+
const name = record["name"];
|
|
489
|
+
if (typeof name === "string"
|
|
490
|
+
&& isCredentialName(name)
|
|
491
|
+
&& isLiteralSecretValue(record["value"])
|
|
492
|
+
&& !full()) {
|
|
493
|
+
const shownName = name.length > 80 ? `${name.slice(0, 80)}...` : name;
|
|
494
|
+
findings.push({ member, rule: "list_variable", keyPath: path, field: shownName, });
|
|
495
|
+
}
|
|
496
|
+
for (const [key, value,] of Object.entries(record)) {
|
|
497
|
+
if (full())
|
|
498
|
+
return;
|
|
499
|
+
const childPath = `${path}${jsonPathSegment(key)}`;
|
|
500
|
+
const credentialKey = isCredentialName(key);
|
|
501
|
+
const leafInContext = inCredential
|
|
502
|
+
&& NON_SECRET_SUFFIX_TOKENS[keyTokens(key).at(-1) ?? ""] !== true;
|
|
503
|
+
if (typeof value === "string") {
|
|
504
|
+
if ((credentialKey || leafInContext) && isLiteralSecretValue(value)) {
|
|
505
|
+
findings.push({ member, rule: "json_key", keyPath: childPath, field: key, });
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
else {
|
|
509
|
+
visit(value, childPath, inCredential || credentialKey, key, depth + 1);
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
};
|
|
513
|
+
visit(parsed, "$", false, "", 0);
|
|
514
|
+
return findings;
|
|
515
|
+
}
|
|
516
|
+
/** Visible warning for a scannable member that could not be scanned for residue and credentials. */
|
|
517
|
+
function residueScanSkippedIssue(member) {
|
|
518
|
+
return {
|
|
519
|
+
severity: "warning",
|
|
520
|
+
code: "residue_scan_skipped",
|
|
521
|
+
message: `member exceeds the ${MAX_RESIDUE_SCAN_BYTES}-byte residue scan limit and was not`
|
|
522
|
+
+ " scanned for credentials, identity or machine-specific residue",
|
|
523
|
+
member,
|
|
524
|
+
};
|
|
525
|
+
}
|
|
526
|
+
/** Deterministic inventory ordering: member, then key path. */
|
|
527
|
+
function sortCredentialFindings(findings) {
|
|
528
|
+
findings.sort((a, b) => a.member.localeCompare(b.member)
|
|
529
|
+
|| (a.keyPath ?? "").localeCompare(b.keyPath ?? "")
|
|
530
|
+
|| a.rule.localeCompare(b.rule));
|
|
531
|
+
}
|
|
348
532
|
/** Deterministic issue ordering: severity, then code, member, message. */
|
|
349
533
|
function sortIssues(issues) {
|
|
350
534
|
issues.sort((a, b) => {
|
|
@@ -413,6 +597,7 @@ export async function inspectProjectArchive(filePath, overrides = {}) {
|
|
|
413
597
|
memberCount: 0,
|
|
414
598
|
valid: false,
|
|
415
599
|
issues,
|
|
600
|
+
credentialFindings: [],
|
|
416
601
|
};
|
|
417
602
|
}
|
|
418
603
|
let zip;
|
|
@@ -434,6 +619,7 @@ export async function inspectProjectArchive(filePath, overrides = {}) {
|
|
|
434
619
|
memberCount: 0,
|
|
435
620
|
valid: false,
|
|
436
621
|
issues,
|
|
622
|
+
credentialFindings: [],
|
|
437
623
|
};
|
|
438
624
|
}
|
|
439
625
|
try {
|
|
@@ -452,6 +638,7 @@ export async function inspectProjectArchive(filePath, overrides = {}) {
|
|
|
452
638
|
memberCount: records.length,
|
|
453
639
|
valid: false,
|
|
454
640
|
issues,
|
|
641
|
+
credentialFindings: [],
|
|
455
642
|
};
|
|
456
643
|
}
|
|
457
644
|
const memberCounts = new Map();
|
|
@@ -481,6 +668,7 @@ export async function inspectProjectArchive(filePath, overrides = {}) {
|
|
|
481
668
|
const recipes = new Map();
|
|
482
669
|
const tally = { expandedBytes: 0, };
|
|
483
670
|
let aggregateAborted = false;
|
|
671
|
+
const credentialFindings = [];
|
|
484
672
|
for (const entry of records) {
|
|
485
673
|
if (!entry.isFile)
|
|
486
674
|
continue;
|
|
@@ -504,11 +692,14 @@ export async function inspectProjectArchive(filePath, overrides = {}) {
|
|
|
504
692
|
});
|
|
505
693
|
}
|
|
506
694
|
const parseCandidate = (isManifest || isRecipeJson) && entry.size <= MAX_JSON_MEMBER_BYTES;
|
|
507
|
-
const
|
|
508
|
-
|| name.startsWith(
|
|
695
|
+
const matchesScanPrefix = isManifest
|
|
696
|
+
|| name.startsWith(`${PROJECT_CONFIG_ROOT}/`)
|
|
509
697
|
|| name.startsWith(`${BUNDLED_DATA_ROOT}/`)
|
|
510
|
-
|| name.startsWith(`${UPLOADS_ROOT}/`)
|
|
511
|
-
|
|
698
|
+
|| name.startsWith(`${UPLOADS_ROOT}/`);
|
|
699
|
+
const scanCandidate = matchesScanPrefix && entry.size <= MAX_RESIDUE_SCAN_BYTES;
|
|
700
|
+
if (matchesScanPrefix && !scanCandidate) {
|
|
701
|
+
issues.push(residueScanSkippedIssue(name));
|
|
702
|
+
}
|
|
512
703
|
// Budgets computed from the central-directory record abort before
|
|
513
704
|
// any decompression work: a member whose declared compressed size
|
|
514
705
|
// or expanded/compressed ratio is above budget is never streamed.
|
|
@@ -621,6 +812,8 @@ export async function inspectProjectArchive(filePath, overrides = {}) {
|
|
|
621
812
|
message: `member ${name}: data exceeds the inspection byte limit and was not buffered for parsing`,
|
|
622
813
|
member: name,
|
|
623
814
|
});
|
|
815
|
+
if (scanCandidate)
|
|
816
|
+
issues.push(residueScanSkippedIssue(name));
|
|
624
817
|
}
|
|
625
818
|
const content = streamed.content;
|
|
626
819
|
if (!content)
|
|
@@ -649,10 +842,24 @@ export async function inspectProjectArchive(filePath, overrides = {}) {
|
|
|
649
842
|
}
|
|
650
843
|
}
|
|
651
844
|
if (scanCandidate && buffer.length <= MAX_RESIDUE_SCAN_BYTES) {
|
|
652
|
-
const
|
|
845
|
+
const text = buffer.toString("utf8");
|
|
846
|
+
const counts = countResidue(text);
|
|
653
847
|
if (counts.credential + counts.email + counts.homePath + counts.metadataField > 0) {
|
|
654
848
|
issues.push(residueIssue(name, counts));
|
|
655
849
|
}
|
|
850
|
+
if (name.startsWith(`${PROJECT_CONFIG_ROOT}/`)) {
|
|
851
|
+
const found = collectCredentialFindings(name, text);
|
|
852
|
+
if (found.length > 0) {
|
|
853
|
+
credentialFindings.push(...found);
|
|
854
|
+
issues.push({
|
|
855
|
+
severity: "warning",
|
|
856
|
+
code: "credential_structure",
|
|
857
|
+
message: `member contains ${found.length} credential-bearing structure(s);`
|
|
858
|
+
+ " see the credential inventory for key paths (values are never reported)",
|
|
859
|
+
member: name,
|
|
860
|
+
});
|
|
861
|
+
}
|
|
862
|
+
}
|
|
656
863
|
}
|
|
657
864
|
}
|
|
658
865
|
// Once the aggregate expanded-byte budget is crossed, remaining
|
|
@@ -660,6 +867,7 @@ export async function inspectProjectArchive(filePath, overrides = {}) {
|
|
|
660
867
|
// manifest/graph conclusion is drawn from the partial inspection.
|
|
661
868
|
if (aggregateAborted) {
|
|
662
869
|
sortIssues(issues);
|
|
870
|
+
sortCredentialFindings(credentialFindings);
|
|
663
871
|
return {
|
|
664
872
|
filePath,
|
|
665
873
|
sizeBytes,
|
|
@@ -667,6 +875,7 @@ export async function inspectProjectArchive(filePath, overrides = {}) {
|
|
|
667
875
|
sourceProjectKey: undefined,
|
|
668
876
|
valid: false,
|
|
669
877
|
issues,
|
|
878
|
+
credentialFindings,
|
|
670
879
|
};
|
|
671
880
|
}
|
|
672
881
|
let sourceProjectKey;
|
|
@@ -726,6 +935,7 @@ export async function inspectProjectArchive(filePath, overrides = {}) {
|
|
|
726
935
|
checkRecipeReferences(recipes, datasetNames, nonDatasetNames, issues);
|
|
727
936
|
checkOrphanMembers(records, recipeNames, anchored, issues);
|
|
728
937
|
sortIssues(issues);
|
|
938
|
+
sortCredentialFindings(credentialFindings);
|
|
729
939
|
return {
|
|
730
940
|
filePath,
|
|
731
941
|
sizeBytes,
|
|
@@ -733,6 +943,7 @@ export async function inspectProjectArchive(filePath, overrides = {}) {
|
|
|
733
943
|
sourceProjectKey,
|
|
734
944
|
valid: !issues.some((issue) => issue.severity === "error"),
|
|
735
945
|
issues,
|
|
946
|
+
credentialFindings,
|
|
736
947
|
};
|
|
737
948
|
}
|
|
738
949
|
finally {
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { chmod, rename, rm, stat, } from "node:fs/promises";
|
|
1
|
+
import { chmod, mkdir, open, rename, rm, stat, } from "node:fs/promises";
|
|
3
2
|
import { basename, dirname, join, } from "node:path";
|
|
4
3
|
/** Stream with backpressure; publish only a complete download, preserving existing output on failure. */
|
|
5
4
|
export async function writeResponseToFile(path, response, permissions = "private") {
|
|
@@ -15,10 +14,13 @@ export async function writeResponseToFile(path, response, permissions = "private
|
|
|
15
14
|
}
|
|
16
15
|
}
|
|
17
16
|
const temporaryPath = join(dirname(path), `.${basename(path)}.tmp-${crypto.randomUUID()}`);
|
|
18
|
-
// Exclusive, private temp file
|
|
19
|
-
|
|
17
|
+
// Exclusive, private temp file. The body is copied chunk by chunk with an
|
|
18
|
+
// awaited write per chunk (backpressure). Not Bun.write(Bun.file(fd), response):
|
|
19
|
+
// on Windows, Bun 1.4.3 let the process exit 0 mid-copy with nothing written.
|
|
20
|
+
let handle;
|
|
20
21
|
try {
|
|
21
|
-
|
|
22
|
+
await mkdir(dirname(path), { recursive: true, });
|
|
23
|
+
handle = await open(temporaryPath, "wx", 0o600);
|
|
22
24
|
}
|
|
23
25
|
catch (error) {
|
|
24
26
|
// Nothing will read the body; release the connection before surfacing the error.
|
|
@@ -26,12 +28,17 @@ export async function writeResponseToFile(path, response, permissions = "private
|
|
|
26
28
|
throw error;
|
|
27
29
|
}
|
|
28
30
|
try {
|
|
29
|
-
let bytes;
|
|
31
|
+
let bytes = 0;
|
|
30
32
|
try {
|
|
31
|
-
|
|
33
|
+
if (response.body) {
|
|
34
|
+
for await (const chunk of response.body) {
|
|
35
|
+
await handle.write(chunk);
|
|
36
|
+
bytes += chunk.byteLength;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
32
39
|
}
|
|
33
40
|
finally {
|
|
34
|
-
|
|
41
|
+
await handle.close();
|
|
35
42
|
}
|
|
36
43
|
// Keep partial contents private; restore publication permissions only after completion.
|
|
37
44
|
if (mode !== undefined)
|
|
@@ -7,8 +7,10 @@
|
|
|
7
7
|
* - optionally redacts userinfo embedded in HTTP(S) URL authorities
|
|
8
8
|
* (`https://user:pass@host/...` -> `https://[redacted]@host/...`).
|
|
9
9
|
*
|
|
10
|
-
* The SDK resource layer forwards values verbatim to DSS over TLS;
|
|
11
|
-
* is a CLI presentation concern and must never leak secret
|
|
10
|
+
* The SDK resource layer forwards values verbatim to DSS over TLS; sanitizing
|
|
11
|
+
* structured output is a CLI presentation concern and must never leak secret
|
|
12
|
+
* material instead. DSS-produced read text that consumers never need secrets
|
|
13
|
+
* from (job logs) is scrubbed at the SDK download via `redactCredentialPairs`.
|
|
12
14
|
*/
|
|
13
15
|
/** Marker replacing every redacted value or secret occurrence. */
|
|
14
16
|
export declare const SECRET_REDACTED = "[redacted]";
|
|
@@ -46,6 +48,13 @@ export declare function redactUrlUserinfo(text: string): string;
|
|
|
46
48
|
* recurse, arrays recurse. Non-string scalars pass through unchanged.
|
|
47
49
|
*/
|
|
48
50
|
export declare function sanitizeSecrets<T>(value: T, options: SecretSanitizeOptions): T;
|
|
51
|
+
/**
|
|
52
|
+
* Redact the string value of every JSON-style `"key":"value"` pair in free
|
|
53
|
+
* text whose key is credential-like (e.g. the `jobTicketSecret` DSS writes
|
|
54
|
+
* into job logs). Bare `key` fields are not redacted here: in log text they
|
|
55
|
+
* name ordinary identifiers far more often than secrets.
|
|
56
|
+
*/
|
|
57
|
+
export declare function redactCredentialPairs(text: string): string;
|
|
49
58
|
/**
|
|
50
59
|
* Every string value under a credential-like key (any depth), plus bare
|
|
51
60
|
* `key` fields such as `authRealm.key`. Collected from a command's own JSON
|
|
@@ -7,8 +7,10 @@
|
|
|
7
7
|
* - optionally redacts userinfo embedded in HTTP(S) URL authorities
|
|
8
8
|
* (`https://user:pass@host/...` -> `https://[redacted]@host/...`).
|
|
9
9
|
*
|
|
10
|
-
* The SDK resource layer forwards values verbatim to DSS over TLS;
|
|
11
|
-
* is a CLI presentation concern and must never leak secret
|
|
10
|
+
* The SDK resource layer forwards values verbatim to DSS over TLS; sanitizing
|
|
11
|
+
* structured output is a CLI presentation concern and must never leak secret
|
|
12
|
+
* material instead. DSS-produced read text that consumers never need secrets
|
|
13
|
+
* from (job logs) is scrubbed at the SDK download via `redactCredentialPairs`.
|
|
12
14
|
*/
|
|
13
15
|
/** Marker replacing every redacted value or secret occurrence. */
|
|
14
16
|
export const SECRET_REDACTED = "[redacted]";
|
|
@@ -92,6 +94,24 @@ const CREDENTIAL_KEY_FRAGMENTS = [
|
|
|
92
94
|
];
|
|
93
95
|
/** Keys naming how credentials work (`credentialsMode`, `passwordType`), not secrets. */
|
|
94
96
|
const CREDENTIAL_SETTING_SUFFIXES = ["mode", "type", "policy",];
|
|
97
|
+
/** Credential-like key (normalized fragment match, minus credential settings). */
|
|
98
|
+
function isCredentialKey(key) {
|
|
99
|
+
const normalized = normalizeSecretKey(key);
|
|
100
|
+
if (CREDENTIAL_SETTING_SUFFIXES.some((suffix) => normalized.endsWith(suffix)))
|
|
101
|
+
return false;
|
|
102
|
+
return CREDENTIAL_KEY_FRAGMENTS.some((fragment) => normalized.includes(fragment));
|
|
103
|
+
}
|
|
104
|
+
/** JSON-style `"key": "value"` pair with escape-aware string bodies. */
|
|
105
|
+
const JSON_STRING_PAIR = /"((?:[^"\\\n]|\\.)*)"(\s*:\s*)"((?:[^"\\\n]|\\.)*)"/g;
|
|
106
|
+
/**
|
|
107
|
+
* Redact the string value of every JSON-style `"key":"value"` pair in free
|
|
108
|
+
* text whose key is credential-like (e.g. the `jobTicketSecret` DSS writes
|
|
109
|
+
* into job logs). Bare `key` fields are not redacted here: in log text they
|
|
110
|
+
* name ordinary identifiers far more often than secrets.
|
|
111
|
+
*/
|
|
112
|
+
export function redactCredentialPairs(text) {
|
|
113
|
+
return text.replace(JSON_STRING_PAIR, (pair, key, separator, value) => value.length > 0 && isCredentialKey(key) ? `"${key}"${separator}"${SECRET_REDACTED}"` : pair);
|
|
114
|
+
}
|
|
95
115
|
/**
|
|
96
116
|
* Every string value under a credential-like key (any depth), plus bare
|
|
97
117
|
* `key` fields such as `authRealm.key`. Collected from a command's own JSON
|
|
@@ -114,10 +134,7 @@ export function credentialValues(value) {
|
|
|
114
134
|
if (node === null || typeof node !== "object")
|
|
115
135
|
return;
|
|
116
136
|
for (const [key, item,] of Object.entries(node)) {
|
|
117
|
-
|
|
118
|
-
const setting = CREDENTIAL_SETTING_SUFFIXES.some((suffix) => normalized.endsWith(suffix));
|
|
119
|
-
visit(item, sensitive || (!setting && (normalized === "key"
|
|
120
|
-
|| CREDENTIAL_KEY_FRAGMENTS.some((fragment) => normalized.includes(fragment)))));
|
|
137
|
+
visit(item, sensitive || isCredentialKey(key) || normalizeSecretKey(key) === "key");
|
|
121
138
|
}
|
|
122
139
|
};
|
|
123
140
|
visit(value, false);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dataiku-sdk",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "4.0.0",
|
|
4
4
|
"description": "Dataiku DSS SDK and CLI for programmatic access to DSS REST APIs",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -27,40 +27,39 @@
|
|
|
27
27
|
"build": "bun scripts/generate-action-output-schemas.mjs && tsc && bun scripts/generate-build-metadata.mjs",
|
|
28
28
|
"prepack": "bun scripts/sync-plugin-version.mjs --check && bun scripts/generate-action-output-schemas.mjs --check && tsc && bun scripts/generate-build-metadata.mjs",
|
|
29
29
|
"prepublishOnly": "bun run prepack",
|
|
30
|
-
"
|
|
30
|
+
"verify": "bun check -p tsconfig.json && bun check -p tools/oxlint/tsconfig.json && bun run check:live && bun scripts/generate-action-output-schemas.mjs --check && bun scripts/sync-plugin-version.mjs --check && bun scripts/cut-changelog.mjs --check",
|
|
31
31
|
"version": "bun scripts/sync-plugin-version.mjs && bun scripts/cut-changelog.mjs && git add plugin.json CHANGELOG.md",
|
|
32
32
|
"lint": "oxlint src/ tools/oxlint/ scripts/ tests/",
|
|
33
33
|
"lint:fix": "oxlint --fix src/ tools/oxlint/ scripts/ tests/",
|
|
34
34
|
"format": "dprint fmt",
|
|
35
35
|
"format:check": "dprint check",
|
|
36
|
-
"test": "bun test",
|
|
36
|
+
"test": "bun test --parallel --timeout 30000",
|
|
37
37
|
"test:live": "bun --no-env-file scripts/live-suite.ts",
|
|
38
|
-
"check:live": "
|
|
38
|
+
"check:live": "bun check -p tsconfig.live.json",
|
|
39
39
|
"test:tokens": "bun test tests/cli/agent-token-budget.test.ts",
|
|
40
40
|
"test:platform": "bun scripts/platform-smoke.mjs",
|
|
41
|
-
"test:integration": "
|
|
42
|
-
"test:integration:mutating": "
|
|
43
|
-
"test:integration:rigorous": "
|
|
44
|
-
"test:integration:rigorous:mutating": "
|
|
41
|
+
"test:integration": "RUN_DATAIKU_INTEGRATION=1 bun test tests/integration-playground.test.ts",
|
|
42
|
+
"test:integration:mutating": "RUN_DATAIKU_INTEGRATION=1 RUN_DATAIKU_INTEGRATION_MUTATING=1 bun test tests/integration-playground.test.ts",
|
|
43
|
+
"test:integration:rigorous": "RUN_DATAIKU_INTEGRATION=1 bun test tests/integration-rigorous.test.ts",
|
|
44
|
+
"test:integration:rigorous:mutating": "RUN_DATAIKU_INTEGRATION=1 RUN_DATAIKU_INTEGRATION_MUTATING=1 bun test tests/integration-rigorous.test.ts"
|
|
45
45
|
},
|
|
46
46
|
"dependencies": {
|
|
47
47
|
"@sinclair/typebox": "^0.34.52",
|
|
48
48
|
"node-stream-zip": "^1.16.0"
|
|
49
49
|
},
|
|
50
50
|
"devDependencies": {
|
|
51
|
-
"@oxlint/plugins": "1.
|
|
52
|
-
"@types/bun": "^1.4.
|
|
53
|
-
"
|
|
54
|
-
"dprint": "^0.60.1",
|
|
51
|
+
"@oxlint/plugins": "1.87.0",
|
|
52
|
+
"@types/bun": "^1.4.3",
|
|
53
|
+
"dprint": "^0.63.0",
|
|
55
54
|
"js-tiktoken": "1.0.21",
|
|
56
|
-
"oxlint": "1.
|
|
55
|
+
"oxlint": "1.87.0",
|
|
57
56
|
"typescript": "^7.0.2"
|
|
58
57
|
},
|
|
59
58
|
"bundleDependencies": [
|
|
60
59
|
"@sinclair/typebox"
|
|
61
60
|
],
|
|
62
61
|
"engines": {
|
|
63
|
-
"bun": ">=1.4.
|
|
62
|
+
"bun": ">=1.4.3"
|
|
64
63
|
},
|
|
65
64
|
"publishConfig": {
|
|
66
65
|
"access": "public"
|
package/plugin.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json",
|
|
3
3
|
"name": "dataiku-sdk",
|
|
4
|
-
"version": "
|
|
4
|
+
"version": "4.0.0",
|
|
5
5
|
"description": "Dataiku DSS SDK and agent-oriented CLI for programmatic access to DSS REST APIs.",
|
|
6
6
|
"homepage": "https://github.com/clssck/dataiku-sdk",
|
|
7
7
|
"repository": "https://github.com/clssck/dataiku-sdk",
|
|
@@ -6,7 +6,7 @@ description: >-
|
|
|
6
6
|
Discover syntax: dss commands run --fields RESOURCE.ACTION.
|
|
7
7
|
license: LicenseRef-Dataiku-SDK-Limited-Use-1.0
|
|
8
8
|
compatibility: >-
|
|
9
|
-
Requires Bun >= 1.4.
|
|
9
|
+
Requires Bun >= 1.4.3, the CLI, DSS network access, and credentials.
|
|
10
10
|
---
|
|
11
11
|
|
|
12
12
|
# Dataiku DSS
|
|
@@ -3,12 +3,12 @@
|
|
|
3
3
|
## Contract
|
|
4
4
|
|
|
5
5
|
- Stdout: exactly one compact JSON value; void success: `{ok:true}`. No prompts, help screens, tables, banners, or prose.
|
|
6
|
-
- Dispatch/runtime failure: stdout error object (`type:"error"`, `
|
|
6
|
+
- Dispatch/runtime failure: stdout error object (`type:"error"`, `error`, `code`, `category`, `exitCode`; `projectKey` when resolved), nonzero exit. `doctor`/`batch`/`cleanup` failures instead return direct result objects; inspect exit code and fields.
|
|
7
7
|
- Stderr: JSONL diagnostics only; `type:"warning"` / `type:"trace"` for warnings / `--verbose` HTTP traces, flushed before success or failure output.
|
|
8
8
|
- Exits: 0 success, 1 usage/configuration, 2 DSS/internal/permission-or-environment, 3 transient/retryable DSS error, 4 failed long-running result/assertion.
|
|
9
9
|
- Recipe payload stdout: JSON string. With `--output PATH`: exact bytes to file, JSON string equal to `PATH` on stdout.
|
|
10
10
|
- General `--fields a,b,c`: object projection, element-wise for object arrays. Dotted paths traverse nested objects; missing fields become `null`; strings/scalars pass through.
|
|
11
|
-
- `*.list
|
|
11
|
+
- Compact by default: `*.list` (ids, kind, next-step fields such as recipe `inputs`/`outputs`), `job get` (no log tail), `dataset get`/`info`, `project get`; `--full`/`--fields` use DSS objects. Lists: `--contains TEXT` filters ids; `--limit N` caps (`list_truncated` warning).
|
|
12
12
|
|
|
13
13
|
## Discovery
|
|
14
14
|
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
## Platform, builds, data
|
|
4
4
|
|
|
5
5
|
- Code/SQL: use `--file`/`--sql-file`; shells, especially PowerShell, mangle quotes, `$`, and multiline text. Non-UTF-8 consoles (e.g. Windows cp1252): write non-ASCII results to UTF-8 files or `--output PATH`, not the console.
|
|
6
|
-
- Build errors: `dss job log <id> --errors-only` selects errors/tracebacks.
|
|
6
|
+
- Build errors: `dss job log <id> --errors-only` selects errors/tracebacks. Plain `job log` prints the last 100 lines (`log_truncated`); for full logs, use `--output PATH` without `--errors-only`. Logs may arrive as one JVM-noisy line; `Error in Python process: At line <N>` identifies the payload source line.
|
|
7
7
|
- Schemas: code recipes set output schemas at run time. Visual/SQL query recipes: `recipe create` fills created or empty outputs (`outputSchemaUpdated`); `recipe set-payload` updates empty or DSS-derived ones, warns `recipe_output_schema_outdated` for hand-edited ones. Apply with `dss recipe update-schema NAME` (one recipe), `dss flow propagate-schema DATASET --wait` (every downstream recipe), or build `--auto-update-schema`. `built_dataset_has_no_columns`: the DONE build wrote nothing. `nothing_to_build` (exit 4): no recipe writes the managed target. Manual columns: `dss dataset refresh-schema NAME --data-file columns.json`. `dss dataset validate-build` catches file-backed misconfiguration.
|
|
8
8
|
- `Inline` (editable) datasets: the public API cannot write rows, so `dataset create` rejects them; use `UploadedFiles` + `dataset upload-file`, or a python recipe output. `code run` is a scenario step; some DSS versions refuse dataset writes there.
|
|
9
9
|
- `dss dataset download`: default 100k rows; result `{ path, rows, truncated, limit }`. If truncated, raise `--limit`. Specify `--output` or read the `dataset_download_default_location` path warning. CSV is spreadsheet-safe; `--raw-data` retains formula prefixes. For large tables, aggregate in SQL or use a recipe.
|
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
|
|
23
23
|
Dispatch/runtime failure: one compact stdout error:
|
|
24
24
|
```json
|
|
25
|
-
{"type":"error","
|
|
25
|
+
{"type":"error","error":"Missing API key.","code":"missing_required_flag","category":"usage","exitCode":1}
|
|
26
26
|
```
|
|
27
27
|
|
|
28
28
|
`doctor`/`batch`/`cleanup` command failures return direct stdout result objects; check nonzero exit before interpreting. Recover from structured `code`, `category`, `exitCode`, `retryable`, `status`, `details`, never message scraping.
|