dataiku-sdk 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -1
- package/bin/dss.js +20 -1
- package/dist/build-metadata.json +1 -0
- package/dist/src/cli/commands/data-quality.js +42 -0
- package/dist/src/cli/commands/dataset.d.ts +23 -0
- package/dist/src/cli/commands/dataset.js +187 -6
- package/dist/src/cli/commands/job.js +15 -2
- package/dist/src/cli/commands/project.js +116 -12
- package/dist/src/cli/contract.d.ts +1 -0
- package/dist/src/cli/contract.js +182 -13
- package/dist/src/cli/doctor.d.ts +12 -2
- package/dist/src/cli/doctor.js +44 -13
- package/dist/src/cli/flags.js +2 -0
- package/dist/src/cli/helpers/cleanup.js +30 -1
- package/dist/src/cli/output.d.ts +3 -1
- package/dist/src/cli/output.js +11 -6
- package/dist/src/cli/types.d.ts +2 -0
- package/dist/src/cli/version.d.ts +46 -4
- package/dist/src/cli/version.js +175 -9
- package/dist/src/cli.js +51 -13
- package/dist/src/client.d.ts +28 -1
- package/dist/src/client.js +34 -7
- package/dist/src/index.d.ts +3 -2
- package/dist/src/index.js +1 -0
- package/dist/src/resources/datasets.d.ts +43 -0
- package/dist/src/resources/datasets.js +199 -28
- package/dist/src/resources/projects.d.ts +36 -3
- package/dist/src/resources/projects.js +127 -16
- package/dist/src/utils/project-archive.d.ts +32 -0
- package/dist/src/utils/project-archive.js +624 -0
- package/package.json +6 -5
- package/plugin.json +1 -1
package/dist/src/cli/version.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { readFileSync, } from "node:fs";
|
|
1
|
+
import { readdirSync, readFileSync, statSync, } from "node:fs";
|
|
2
2
|
import { dirname, resolve, } from "node:path";
|
|
3
3
|
import { fileURLToPath, } from "node:url";
|
|
4
4
|
export function findPackageRoot() {
|
|
@@ -36,22 +36,174 @@ export function gitDirectory(packageRoot) {
|
|
|
36
36
|
}
|
|
37
37
|
return resolve(packageRoot, ".git");
|
|
38
38
|
}
|
|
39
|
+
/**
|
|
40
|
+
* A full lowercase hexadecimal revision is the only provenance claim trusted.
|
|
41
|
+
* Git is never invoked from runtime version reporting; revisions are resolved
|
|
42
|
+
* directly from the filesystem layouts Git documents.
|
|
43
|
+
*/
|
|
44
|
+
const FULL_REVISION_PATTERN = /^[0-9a-f]{40}$/;
|
|
45
|
+
export function isFullRevision(value) {
|
|
46
|
+
return typeof value === "string" && FULL_REVISION_PATTERN.test(value);
|
|
47
|
+
}
|
|
48
|
+
function readGitFile(base, relative) {
|
|
49
|
+
try {
|
|
50
|
+
return readFileSync(resolve(base, relative), "utf-8").trim();
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
return undefined;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
function isSafeRefName(ref) {
|
|
57
|
+
return (ref.length > 0
|
|
58
|
+
&& !ref.startsWith("/")
|
|
59
|
+
&& !ref.endsWith("/")
|
|
60
|
+
&& !ref.includes("\\")
|
|
61
|
+
&& !ref.split("/").includes(".."));
|
|
62
|
+
}
|
|
63
|
+
/** Resolve an entry from packed-refs; peeled `^` lines never match a ref. */
|
|
64
|
+
function packedRefFile(base, ref) {
|
|
65
|
+
let contents;
|
|
66
|
+
try {
|
|
67
|
+
contents = readFileSync(resolve(base, "packed-refs"), "utf-8");
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
return undefined;
|
|
71
|
+
}
|
|
72
|
+
for (const line of contents.split("\n")) {
|
|
73
|
+
const trimmed = line.trim();
|
|
74
|
+
if (trimmed.length === 0 || trimmed.startsWith("#") || trimmed.startsWith("^"))
|
|
75
|
+
continue;
|
|
76
|
+
const splitAt = trimmed.indexOf(" ");
|
|
77
|
+
if (splitAt <= 0)
|
|
78
|
+
continue;
|
|
79
|
+
const hash = trimmed.slice(0, splitAt);
|
|
80
|
+
if (isFullRevision(hash) && trimmed.slice(splitAt + 1).trim() === ref)
|
|
81
|
+
return hash;
|
|
82
|
+
}
|
|
83
|
+
return undefined;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* The shared git dir behind a linked worktree or submodule checkout: the
|
|
87
|
+
* `commondir` file inside the worktree's git dir, resolved relative to that
|
|
88
|
+
* dir. Normal checkouts (no `commondir` file) share their own git dir.
|
|
89
|
+
*/
|
|
90
|
+
function gitCommonDirectory(gitDir) {
|
|
91
|
+
try {
|
|
92
|
+
const commondir = readFileSync(resolve(gitDir, "commondir"), "utf-8").trim();
|
|
93
|
+
return commondir.length > 0 ? resolve(gitDir, commondir) : undefined;
|
|
94
|
+
}
|
|
95
|
+
catch {
|
|
96
|
+
return undefined;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Full checkout HEAD revision (40-lowercase-hex) — the basis for staleness
|
|
101
|
+
* comparison. Resolution follows Git's own layouts: a detached revision, then
|
|
102
|
+
* loose refs in the worktree git dir, then the shared store reached through
|
|
103
|
+
* `commondir`, then packed-refs in each. Anything that is not a full
|
|
104
|
+
* lowercase hexadecimal revision is rejected.
|
|
105
|
+
*/
|
|
106
|
+
export function gitFullRevision(packageRoot) {
|
|
107
|
+
if (!packageRoot)
|
|
108
|
+
return undefined;
|
|
109
|
+
const gitDir = gitDirectory(packageRoot);
|
|
110
|
+
const head = readGitFile(gitDir, "HEAD");
|
|
111
|
+
if (!head)
|
|
112
|
+
return undefined;
|
|
113
|
+
if (isFullRevision(head))
|
|
114
|
+
return head;
|
|
115
|
+
if (!head.startsWith("ref:"))
|
|
116
|
+
return undefined;
|
|
117
|
+
const ref = head.slice("ref:".length).trim();
|
|
118
|
+
if (!isSafeRefName(ref))
|
|
119
|
+
return undefined;
|
|
120
|
+
for (const base of [gitDir, gitCommonDirectory(gitDir),]) {
|
|
121
|
+
if (!base)
|
|
122
|
+
continue;
|
|
123
|
+
const loose = readGitFile(base, ref);
|
|
124
|
+
if (loose && isFullRevision(loose))
|
|
125
|
+
return loose;
|
|
126
|
+
const packed = packedRefFile(base, ref);
|
|
127
|
+
if (packed)
|
|
128
|
+
return packed;
|
|
129
|
+
}
|
|
130
|
+
return undefined;
|
|
131
|
+
}
|
|
39
132
|
export function gitRevision(packageRoot) {
|
|
133
|
+
const full = gitFullRevision(packageRoot);
|
|
134
|
+
return full ? full.slice(0, 7) : undefined;
|
|
135
|
+
}
|
|
136
|
+
export function detectLoadSource(modulePath = fileURLToPath(import.meta.url)) {
|
|
137
|
+
return modulePath.replaceAll("\\", "/").includes("/dist/") ? "dist" : "source";
|
|
138
|
+
}
|
|
139
|
+
export function buildMetadataRevision(packageRoot) {
|
|
40
140
|
if (!packageRoot)
|
|
41
141
|
return undefined;
|
|
42
142
|
try {
|
|
43
|
-
const
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
const full = readFileSync(resolve(gitDir, ref), "utf-8").trim();
|
|
49
|
-
return full.slice(0, 7);
|
|
143
|
+
const metadata = JSON.parse(readFileSync(resolve(packageRoot, "dist", "build-metadata.json"), "utf-8"));
|
|
144
|
+
return typeof metadata.buildRevision === "string"
|
|
145
|
+
&& isFullRevision(metadata.buildRevision)
|
|
146
|
+
? metadata.buildRevision
|
|
147
|
+
: undefined;
|
|
50
148
|
}
|
|
51
149
|
catch {
|
|
52
150
|
return undefined;
|
|
53
151
|
}
|
|
54
152
|
}
|
|
153
|
+
/**
|
|
154
|
+
* Whether compiled CLI output predates any source input. The build metadata
|
|
155
|
+
* file is written after `tsc`, so its mtime is the build-completion marker.
|
|
156
|
+
* Installed packages omit `src/`; without both sides there is no drift claim.
|
|
157
|
+
*/
|
|
158
|
+
export function sourceTreeNewerThanBuild(packageRoot) {
|
|
159
|
+
if (!packageRoot)
|
|
160
|
+
return false;
|
|
161
|
+
try {
|
|
162
|
+
const buildMtimeMs = statSync(resolve(packageRoot, "dist", "build-metadata.json")).mtimeMs;
|
|
163
|
+
const pending = [resolve(packageRoot, "src"),];
|
|
164
|
+
while (pending.length > 0) {
|
|
165
|
+
const path = pending.pop();
|
|
166
|
+
if (!path)
|
|
167
|
+
break;
|
|
168
|
+
const stat = statSync(path);
|
|
169
|
+
if (stat.mtimeMs > buildMtimeMs)
|
|
170
|
+
return true;
|
|
171
|
+
if (!stat.isDirectory())
|
|
172
|
+
continue;
|
|
173
|
+
for (const entry of readdirSync(path, { withFileTypes: true, })) {
|
|
174
|
+
if (entry.isDirectory() || entry.isFile()) {
|
|
175
|
+
pending.push(resolve(path, entry.name));
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
return false;
|
|
180
|
+
}
|
|
181
|
+
catch {
|
|
182
|
+
return false;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
export function detectRuntime() {
|
|
186
|
+
return process.versions.bun ? "bun" : "node";
|
|
187
|
+
}
|
|
188
|
+
export function buildVersionPayload(input) {
|
|
189
|
+
const isDist = input.loadSource === "dist";
|
|
190
|
+
// Source execution never claims a build revision, even if the environment
|
|
191
|
+
// happens to carry one.
|
|
192
|
+
const buildRevision = isDist && input.buildRevision ? input.buildRevision : null;
|
|
193
|
+
const staleBuild = isDist
|
|
194
|
+
&& ((buildRevision !== null
|
|
195
|
+
&& input.checkoutFullRevision !== undefined
|
|
196
|
+
&& input.checkoutFullRevision !== buildRevision)
|
|
197
|
+
|| input.sourceNewerThanBuild === true);
|
|
198
|
+
return {
|
|
199
|
+
version: input.packageVersion,
|
|
200
|
+
gitRevision: input.checkoutRevision ?? null,
|
|
201
|
+
source: isDist ? "dist" : "source",
|
|
202
|
+
runtime: input.runtime,
|
|
203
|
+
buildRevision,
|
|
204
|
+
staleBuild,
|
|
205
|
+
};
|
|
206
|
+
}
|
|
55
207
|
export const PACKAGE_ROOT = findPackageRoot();
|
|
56
208
|
export const CLI_VERSION = packageVersion(PACKAGE_ROOT);
|
|
57
209
|
export const CLI_GIT_REVISION = gitRevision(PACKAGE_ROOT);
|
|
@@ -59,5 +211,19 @@ export const AGENT_CONTRACT_VERSION = 2;
|
|
|
59
211
|
export const JSON_SCHEMA_DRAFT = "https://json-schema.org/draft/2020-12/schema";
|
|
60
212
|
export const AGENT_CONTRACT_SCHEMA_ID = "https://clssck.github.io/dataiku-sdk/schemas/agent-contract-v2.json";
|
|
61
213
|
export function cliVersionResult() {
|
|
62
|
-
|
|
214
|
+
const loadSource = process.env["DSS_LOAD_SOURCE"] ?? detectLoadSource();
|
|
215
|
+
const envBuildRevision = process.env["DSS_BUILD_REVISION"];
|
|
216
|
+
return buildVersionPayload({
|
|
217
|
+
packageVersion: CLI_VERSION,
|
|
218
|
+
checkoutRevision: CLI_GIT_REVISION,
|
|
219
|
+
checkoutFullRevision: gitFullRevision(PACKAGE_ROOT),
|
|
220
|
+
loadSource,
|
|
221
|
+
buildRevision: isFullRevision(envBuildRevision)
|
|
222
|
+
? envBuildRevision
|
|
223
|
+
: (loadSource === "dist" ? buildMetadataRevision(PACKAGE_ROOT) : undefined),
|
|
224
|
+
sourceNewerThanBuild: loadSource === "dist"
|
|
225
|
+
? sourceTreeNewerThanBuild(PACKAGE_ROOT)
|
|
226
|
+
: false,
|
|
227
|
+
runtime: detectRuntime(),
|
|
228
|
+
});
|
|
63
229
|
}
|
package/dist/src/cli.js
CHANGED
|
@@ -32,19 +32,28 @@ import { canonicalDssUrl, } from "./utils/dss-url.js";
|
|
|
32
32
|
// ---------------------------------------------------------------------------
|
|
33
33
|
// Auth commands (run before client creation)
|
|
34
34
|
// ---------------------------------------------------------------------------
|
|
35
|
-
function
|
|
35
|
+
function findUnboundLifecycleCleanup(entries) {
|
|
36
36
|
for (const [index, entry,] of entries.entries()) {
|
|
37
37
|
const parsed = parseArgs(entry.cleanup.argv);
|
|
38
38
|
const [resource, action,] = parsed.positional;
|
|
39
|
-
if (resource
|
|
39
|
+
if (resource === "app" && action === "delete-instance") {
|
|
40
|
+
if (parsed.flags["unconfirmed-creation"] === true)
|
|
41
|
+
continue;
|
|
42
|
+
const expected = parsed.flags["expect-project-incarnation"];
|
|
43
|
+
if (typeof expected !== "string")
|
|
44
|
+
return { index, resource, action, reason: "missing", };
|
|
45
|
+
if (!/^[0-9a-f]{64}$/.test(expected))
|
|
46
|
+
return { index, resource, action, reason: "invalid", };
|
|
40
47
|
continue;
|
|
41
|
-
|
|
48
|
+
}
|
|
49
|
+
if (resource === "project" && action === "delete") {
|
|
50
|
+
const expected = parsed.flags["expect-project-incarnation"];
|
|
51
|
+
if (typeof expected !== "string")
|
|
52
|
+
return { index, resource, action, reason: "missing", };
|
|
53
|
+
if (!/^[0-9a-f]{64}$/.test(expected))
|
|
54
|
+
return { index, resource, action, reason: "invalid", };
|
|
42
55
|
continue;
|
|
43
|
-
|
|
44
|
-
if (typeof expected !== "string")
|
|
45
|
-
return { index, reason: "missing", };
|
|
46
|
-
if (!/^[0-9a-f]{64}$/.test(expected))
|
|
47
|
-
return { index, reason: "invalid", };
|
|
56
|
+
}
|
|
48
57
|
}
|
|
49
58
|
return undefined;
|
|
50
59
|
}
|
|
@@ -119,9 +128,10 @@ async function runCleanup(flags) {
|
|
|
119
128
|
};
|
|
120
129
|
}
|
|
121
130
|
// Lifecycle binding is as load-bearing as server binding. Validate the whole
|
|
122
|
-
// reversed plan before step one so a later legacy app
|
|
123
|
-
// earlier cleanup applied and then fail on project-key
|
|
124
|
-
|
|
131
|
+
// reversed plan before step one so a later legacy app or project entry
|
|
132
|
+
// cannot leave an earlier cleanup applied and then fail on project-key
|
|
133
|
+
// reuse protection.
|
|
134
|
+
const lifecycleViolation = findUnboundLifecycleCleanup(ordered);
|
|
125
135
|
if (lifecycleViolation) {
|
|
126
136
|
return {
|
|
127
137
|
result: {
|
|
@@ -129,8 +139,8 @@ async function runCleanup(flags) {
|
|
|
129
139
|
steps,
|
|
130
140
|
lifecycleError: {
|
|
131
141
|
entryIndex: lifecycleViolation.index,
|
|
132
|
-
resource:
|
|
133
|
-
action:
|
|
142
|
+
resource: lifecycleViolation.resource,
|
|
143
|
+
action: lifecycleViolation.action,
|
|
134
144
|
reason: lifecycleViolation.reason,
|
|
135
145
|
},
|
|
136
146
|
},
|
|
@@ -147,6 +157,11 @@ async function runCleanup(flags) {
|
|
|
147
157
|
|| !commands[resource]?.[action]) {
|
|
148
158
|
throw new UsageError(`Invalid cleanup argv: ${entry.cleanup.argv.join(" ")}`);
|
|
149
159
|
}
|
|
160
|
+
if (resource === "project"
|
|
161
|
+
&& action === "delete"
|
|
162
|
+
&& typeof parsed.flags["expect-project-incarnation"] !== "string") {
|
|
163
|
+
throw new UsageError("Project cleanup entry is not bound to a project incarnation. Refusing deletion after possible project-key reuse.", "validation_failed", "Capture a new cleanup entry from the current CLI. Legacy entries cannot safely delete projects.");
|
|
164
|
+
}
|
|
150
165
|
if (resource === "app"
|
|
151
166
|
&& action === "delete-instance"
|
|
152
167
|
&& parsed.flags["unconfirmed-creation"] !== true
|
|
@@ -1194,6 +1209,8 @@ const MISSING_PROJECT_KEY_ERROR_PREFIX = "projectKey is required";
|
|
|
1194
1209
|
function errorExitCode(err) {
|
|
1195
1210
|
if (err instanceof CommandResultFailure)
|
|
1196
1211
|
return err.exitCode;
|
|
1212
|
+
if (err instanceof ClientValidationError && err.code === "ambiguous_outcome")
|
|
1213
|
+
return 2;
|
|
1197
1214
|
if (err instanceof UsageError || err instanceof ClientValidationError)
|
|
1198
1215
|
return 1;
|
|
1199
1216
|
if (err instanceof DataikuError)
|
|
@@ -1205,6 +1222,19 @@ function errorExitCode(err) {
|
|
|
1205
1222
|
function buildErrorReport(err) {
|
|
1206
1223
|
const context = rawCommandContext();
|
|
1207
1224
|
const exitCode = errorExitCode(err);
|
|
1225
|
+
if (err instanceof ClientValidationError && err.code === "ambiguous_outcome") {
|
|
1226
|
+
return {
|
|
1227
|
+
type: "error",
|
|
1228
|
+
ok: false,
|
|
1229
|
+
error: err.message,
|
|
1230
|
+
code: err.code,
|
|
1231
|
+
category: "dss",
|
|
1232
|
+
exitCode,
|
|
1233
|
+
...(err.hint ? { hint: err.hint, } : {}),
|
|
1234
|
+
...(err.details ? { details: err.details, } : {}),
|
|
1235
|
+
...context,
|
|
1236
|
+
};
|
|
1237
|
+
}
|
|
1208
1238
|
if (err instanceof UsageError || err instanceof ClientValidationError) {
|
|
1209
1239
|
return {
|
|
1210
1240
|
type: "error",
|
|
@@ -1346,6 +1376,14 @@ async function main() {
|
|
|
1346
1376
|
writeCommandResult(plan);
|
|
1347
1377
|
return;
|
|
1348
1378
|
}
|
|
1379
|
+
if (actionMeta.localHandler) {
|
|
1380
|
+
const result = await actionMeta.localHandler(args, flags);
|
|
1381
|
+
const failureExitCode = commandFailureExitCode(result);
|
|
1382
|
+
if (failureExitCode !== undefined)
|
|
1383
|
+
throw new CommandResultFailure(result, failureExitCode);
|
|
1384
|
+
writeCommandResult(result);
|
|
1385
|
+
return;
|
|
1386
|
+
}
|
|
1349
1387
|
const { url, apiKey, projectKey, tlsRejectUnauthorized, caCertPath, } = resolveCredentials(flags);
|
|
1350
1388
|
currentCommandContext.projectKey = projectKey;
|
|
1351
1389
|
if (!url) {
|
package/dist/src/client.d.ts
CHANGED
|
@@ -45,6 +45,21 @@ export interface DataikuClientTraceEvent {
|
|
|
45
45
|
elapsedMs?: number;
|
|
46
46
|
detail?: string;
|
|
47
47
|
}
|
|
48
|
+
/**
|
|
49
|
+
* Selected response metadata captured alongside a public API response body.
|
|
50
|
+
* A fixed whitelist of non-sensitive headers: authorization data is never
|
|
51
|
+
* returned. Null means the header was absent.
|
|
52
|
+
*/
|
|
53
|
+
export interface DataikuClientResponseMeta {
|
|
54
|
+
/** `DSS-Version` — version of the DSS backend answering the request. */
|
|
55
|
+
dssVersion: string | null;
|
|
56
|
+
/** `DSS-API-Version` — version of the API server handling the request. */
|
|
57
|
+
dssApiVersion: string | null;
|
|
58
|
+
/** `Date` — origin server timestamp, when present. */
|
|
59
|
+
date: string | null;
|
|
60
|
+
/** Server-provided request id, when present. */
|
|
61
|
+
requestId: string | null;
|
|
62
|
+
}
|
|
48
63
|
export interface DataikuClientConfig {
|
|
49
64
|
/** DSS base URL (e.g. https://dss.example.com) */
|
|
50
65
|
url: string;
|
|
@@ -163,6 +178,15 @@ export declare class DataikuClient {
|
|
|
163
178
|
getRequestTimeoutMs(): number;
|
|
164
179
|
resolveProjectKey(paramValue?: string): string;
|
|
165
180
|
get<T = unknown>(path: string): Promise<T>;
|
|
181
|
+
/**
|
|
182
|
+
* GET returning the parsed JSON body plus selected response metadata
|
|
183
|
+
* (documented DSS version headers, server date, request id). Authorization
|
|
184
|
+
* headers are never included.
|
|
185
|
+
*/
|
|
186
|
+
getWithMetadata<T = unknown>(path: string): Promise<{
|
|
187
|
+
data: T;
|
|
188
|
+
meta: DataikuClientResponseMeta;
|
|
189
|
+
}>;
|
|
166
190
|
getText(path: string): Promise<string>;
|
|
167
191
|
getTextLimited(path: string, maxBytes: number): Promise<{
|
|
168
192
|
text: string;
|
|
@@ -180,10 +204,13 @@ export declare class DataikuClient {
|
|
|
180
204
|
put<T = unknown>(path: string, body: unknown): Promise<T>;
|
|
181
205
|
del(path: string): Promise<void>;
|
|
182
206
|
putVoid(path: string, body: unknown): Promise<void>;
|
|
183
|
-
|
|
207
|
+
private uploadResponse;
|
|
208
|
+
upload(path: string, filePath: string, fileName?: string): Promise<void>;
|
|
209
|
+
uploadJson<T>(path: string, filePath: string, fileName?: string): Promise<T>;
|
|
184
210
|
stream(path: string): Promise<Response>;
|
|
185
211
|
private getHeaders;
|
|
186
212
|
private getAnyHeaders;
|
|
213
|
+
private responseMeta;
|
|
187
214
|
private logTrace;
|
|
188
215
|
/**
|
|
189
216
|
* Validate raw data against a TypeBox schema, throwing on structural mismatch.
|
package/dist/src/client.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { readFileSync, } from "node:fs";
|
|
1
|
+
import { openAsBlob, readFileSync, } from "node:fs";
|
|
2
|
+
import { basename, } from "node:path";
|
|
2
3
|
import { getCACertificates, } from "node:tls";
|
|
3
4
|
import { Value, } from "@sinclair/typebox/value";
|
|
4
5
|
import { safeParseSchema, } from "./schemas.js";
|
|
@@ -360,6 +361,19 @@ export class DataikuClient {
|
|
|
360
361
|
});
|
|
361
362
|
return this.parseJsonResponse(res);
|
|
362
363
|
}
|
|
364
|
+
/**
|
|
365
|
+
* GET returning the parsed JSON body plus selected response metadata
|
|
366
|
+
* (documented DSS version headers, server date, request id). Authorization
|
|
367
|
+
* headers are never included.
|
|
368
|
+
*/
|
|
369
|
+
async getWithMetadata(path) {
|
|
370
|
+
const res = await this.fetchWithRetry(`${this.baseUrl}${path}`, {
|
|
371
|
+
method: "GET",
|
|
372
|
+
headers: this.getHeaders(),
|
|
373
|
+
});
|
|
374
|
+
const data = await this.parseJsonResponse(res);
|
|
375
|
+
return { data, meta: this.responseMeta(res.headers), };
|
|
376
|
+
}
|
|
363
377
|
async getText(path) {
|
|
364
378
|
const res = await this.fetchWithRetry(`${this.baseUrl}${path}`, {
|
|
365
379
|
method: "GET",
|
|
@@ -464,19 +478,23 @@ export class DataikuClient {
|
|
|
464
478
|
body: JSON.stringify(body),
|
|
465
479
|
});
|
|
466
480
|
}
|
|
467
|
-
async
|
|
468
|
-
const { openAsBlob, } = await import("node:fs");
|
|
469
|
-
const { basename, } = await import("node:path");
|
|
481
|
+
async uploadResponse(path, filePath, fileName) {
|
|
470
482
|
const fileBlob = await openAsBlob(filePath);
|
|
471
|
-
const fileName = basename(filePath);
|
|
472
483
|
const formData = new FormData();
|
|
473
|
-
formData.append("file", fileBlob, fileName);
|
|
474
|
-
|
|
484
|
+
formData.append("file", fileBlob, fileName ?? basename(filePath));
|
|
485
|
+
return this.fetchWithRetry(`${this.baseUrl}${path}`, {
|
|
475
486
|
method: "POST",
|
|
476
487
|
headers: { Authorization: `Bearer ${this.apiKey}`, },
|
|
477
488
|
body: formData,
|
|
478
489
|
});
|
|
479
490
|
}
|
|
491
|
+
async upload(path, filePath, fileName) {
|
|
492
|
+
await this.uploadResponse(path, filePath, fileName);
|
|
493
|
+
}
|
|
494
|
+
async uploadJson(path, filePath, fileName) {
|
|
495
|
+
const response = await this.uploadResponse(path, filePath, fileName);
|
|
496
|
+
return this.parseJsonResponse(response);
|
|
497
|
+
}
|
|
480
498
|
async stream(path) {
|
|
481
499
|
return this.fetchWithRetry(`${this.baseUrl}${path}`, {
|
|
482
500
|
method: "GET",
|
|
@@ -497,6 +515,15 @@ export class DataikuClient {
|
|
|
497
515
|
Accept: "*/*",
|
|
498
516
|
};
|
|
499
517
|
}
|
|
518
|
+
responseMeta(headers) {
|
|
519
|
+
// Fixed non-sensitive whitelist. `Headers.get` is case-insensitive.
|
|
520
|
+
return {
|
|
521
|
+
dssVersion: headers.get("dss-version"),
|
|
522
|
+
dssApiVersion: headers.get("dss-api-version"),
|
|
523
|
+
date: headers.get("date"),
|
|
524
|
+
requestId: this.requestIdFromHeaders(headers) ?? null,
|
|
525
|
+
};
|
|
526
|
+
}
|
|
500
527
|
logTrace(event) {
|
|
501
528
|
if (this.verbose)
|
|
502
529
|
this.onTrace({ type: "trace", ...event, });
|
package/dist/src/index.d.ts
CHANGED
|
@@ -12,7 +12,7 @@ export { type ConnectionSchemaListOptions, ConnectionsResource, type ConnectionT
|
|
|
12
12
|
export { ContinuousActivitiesResource, } from "./resources/continuous-activities.js";
|
|
13
13
|
export { DashboardsResource, } from "./resources/dashboards.js";
|
|
14
14
|
export { DataQualityResource, } from "./resources/data-quality.js";
|
|
15
|
-
export { type DatasetBuildValidationResult, type DatasetCloneOptions, type DatasetCloneResult, type DatasetSchemaColumnInput, DatasetsResource, } from "./resources/datasets.js";
|
|
15
|
+
export { type DatasetBuildValidationResult, type DatasetCloneOptions, type DatasetCloneResult, type DatasetSchemaColumnInput, DatasetsResource, type UploadDatasetFileOptions, type UploadDatasetFileResult, type UploadedFileMetadata, } from "./resources/datasets.js";
|
|
16
16
|
export { DiscussionsResource, } from "./resources/discussions.js";
|
|
17
17
|
export { type FlowZoneItemInput, FlowZonesResource, } from "./resources/flow-zones.js";
|
|
18
18
|
export { FoldersResource, } from "./resources/folders.js";
|
|
@@ -26,7 +26,7 @@ export * from "./resources/model-evaluation-stores.js";
|
|
|
26
26
|
export { NotebooksResource, } from "./resources/notebooks.js";
|
|
27
27
|
export * from "./resources/project-git.js";
|
|
28
28
|
export { ProjectLibraryResource, } from "./resources/project-library.js";
|
|
29
|
-
export { type FlowMapResult, ProjectsResource, } from "./resources/projects.js";
|
|
29
|
+
export { type FlowMapResult, type ProjectImportProcessResult, type ProjectImportResult, type ProjectImportSettings, type ProjectImportUploadResult, ProjectsResource, } from "./resources/projects.js";
|
|
30
30
|
export { type RecipeCloneOptions, type RecipeCloneResult, type RecipeGraphReference, type RecipeGraphValidationResult, type RecipeRunOptions, type RecipeRunOutput, type RecipeRunResult, RecipesResource, } from "./resources/recipes.js";
|
|
31
31
|
export * from "./resources/saved-models.js";
|
|
32
32
|
export { normalizeScenarioUpdateData, SCENARIO_CANONICAL_EDITABLE_FIELDS, type ScenarioFieldChange, type ScenarioFieldMismatch, type ScenarioScriptRunResult, ScenariosResource, type ScenarioUpdateNormalization, type ScenarioUpdatePreview, scenarioUpdatePreview, type ScenarioUpdateResult, } from "./resources/scenarios.js";
|
|
@@ -42,5 +42,6 @@ export type { SafeParseResult, } from "./schemas.js";
|
|
|
42
42
|
export type { BuildMode, CodeEnvActionResult, CodeEnvCreateOptions, CodeEnvDetails, CodeEnvPackageList, CodeEnvSetPackagesOptions, CodeEnvSummary, CodeEnvUpdatePackagesOptions, CodeEnvUsage, CodeEnvWaitOptions, ConnectionSummary, DashboardDetails, DashboardSummary, DataQualityComputeResult, DataQualityProjectStatus, DataQualityRule, DataQualityRuleResult, DataQualityRules, DataQualityStatus, DataQualityStatusByPartition, DataQualityTimeline, DataQualityTimelineEntry, DatasetCreateOptions, DatasetDetails, DatasetSchema, DatasetSummary, FlowMapOptions, FlowZone, FlowZoneCreateOptions, FlowZoneItem, FlowZoneObjectType, FlowZonePosition, FlowZoneUpdateOptions, FolderCreateOptions, FolderDetails, FolderItem, FolderSummary, FutureState, FutureWaitResult, InsightDetails, InsightSummary, JobSummary, JobWaitResult, JupyterCell, JupyterNotebookContent, JupyterNotebookSummary, NotebookSession, ProjectDetails, ProjectGitActionResult, ProjectGitDiffResult, ProjectGitFutureResponse, ProjectGitFutureState, ProjectGitLibraries, ProjectGitLibrary, ProjectGitLogResult, ProjectGitRemote, ProjectGitStatus, ProjectGitTag, ProjectGitTags, ProjectMetadata, ProjectSummary, ProjectVariables, RecipeCreateOptions, RecipeCreateResult, RecipeDetails, RecipeSummary, ScenarioDetails, ScenarioStatus, ScenarioStepRun, ScenarioSummary, ScenarioWaitResult, SqlNotebookCell, SqlNotebookContent, SqlNotebookSummary, SqlQueryResponse, SqlQueryResult, SqlQuerySchema, WikiArticleData, WikiArticleMetadata, WikiSettings, WikiTaxonomyNode, } from "./schemas.js";
|
|
43
43
|
export { deepMerge, } from "./utils/deep-merge.js";
|
|
44
44
|
export { type NormalizedFlowEdge, type NormalizedFlowMap, type NormalizedFlowNode, normalizeFlowGraph, } from "./utils/flow-map.js";
|
|
45
|
+
export * from "./utils/project-archive.js";
|
|
45
46
|
export { sanitizeFileName, } from "./utils/sanitize.js";
|
|
46
47
|
export { validateStreamColumns, } from "./resources/datasets.js";
|
package/dist/src/index.js
CHANGED
|
@@ -46,6 +46,7 @@ export { BuildModeSchema, CodeEnvActionResultSchema, CodeEnvCreateOptionsSchema,
|
|
|
46
46
|
// Utilities
|
|
47
47
|
export { deepMerge, } from "./utils/deep-merge.js";
|
|
48
48
|
export { normalizeFlowGraph, } from "./utils/flow-map.js";
|
|
49
|
+
export * from "./utils/project-archive.js";
|
|
49
50
|
export { sanitizeFileName, } from "./utils/sanitize.js";
|
|
50
51
|
// Stream validation
|
|
51
52
|
export { validateStreamColumns, } from "./resources/datasets.js";
|
|
@@ -32,6 +32,25 @@ export interface DatasetCloneResult {
|
|
|
32
32
|
created: Record<string, unknown>;
|
|
33
33
|
settings: Record<string, unknown>;
|
|
34
34
|
}
|
|
35
|
+
export interface UploadedFileMetadata {
|
|
36
|
+
filename: string;
|
|
37
|
+
path?: string;
|
|
38
|
+
length?: number;
|
|
39
|
+
mime?: string;
|
|
40
|
+
weird?: boolean;
|
|
41
|
+
[key: string]: unknown;
|
|
42
|
+
}
|
|
43
|
+
export interface UploadDatasetFileOptions {
|
|
44
|
+
projectKey?: string;
|
|
45
|
+
fileName: string;
|
|
46
|
+
}
|
|
47
|
+
export interface UploadDatasetFileResult {
|
|
48
|
+
datasetName: string;
|
|
49
|
+
projectKey: string;
|
|
50
|
+
fileName: string;
|
|
51
|
+
bytes: number;
|
|
52
|
+
after: UploadedFileMetadata;
|
|
53
|
+
}
|
|
35
54
|
/**
|
|
36
55
|
* Compare streamed TSV header columns against a known dataset schema.
|
|
37
56
|
* Returns an array of warning strings (empty if all columns match).
|
|
@@ -49,6 +68,11 @@ export declare class DatasetsResource extends BaseResource {
|
|
|
49
68
|
schema(datasetName: string, projectKey?: string): Promise<DatasetSchema>;
|
|
50
69
|
/** Replace dataset schema columns directly through the schema endpoint. */
|
|
51
70
|
updateSchema(datasetName: string, columns: DatasetSchemaColumnInput[], projectKey?: string): Promise<void>;
|
|
71
|
+
/**
|
|
72
|
+
* Get the raw dataset schema object exactly as the DSS schema endpoint
|
|
73
|
+
* returns it, without schema validation — used for deterministic comparison.
|
|
74
|
+
*/
|
|
75
|
+
getSchemaObject(datasetName: string, projectKey?: string): Promise<unknown>;
|
|
52
76
|
/**
|
|
53
77
|
* Preview dataset rows as structured data: column names plus row arrays,
|
|
54
78
|
* mirroring the sql query shape ({ columns, rows, rowCount }). Streams TSV
|
|
@@ -70,6 +94,21 @@ export declare class DatasetsResource extends BaseResource {
|
|
|
70
94
|
}>;
|
|
71
95
|
rows: string[][];
|
|
72
96
|
rowCount: number;
|
|
97
|
+
truncated: boolean;
|
|
98
|
+
limit: number;
|
|
99
|
+
}>;
|
|
100
|
+
/**
|
|
101
|
+
* Count dataset rows by streaming the TSV data endpoint and stopping as soon
|
|
102
|
+
* as `expected + 1` rows have been seen, so a mismatched dataset is never
|
|
103
|
+
* fully scanned. Returns the observed count together with `exact` (false when
|
|
104
|
+
* the probe cap was reached, i.e. the count is a lower bound) and whether the
|
|
105
|
+
* count satisfies the assertion.
|
|
106
|
+
*/
|
|
107
|
+
assertRowCount(datasetName: string, expected: number, projectKey?: string): Promise<{
|
|
108
|
+
expected: number;
|
|
109
|
+
count: number;
|
|
110
|
+
exact: boolean;
|
|
111
|
+
satisfied: boolean;
|
|
73
112
|
}>;
|
|
74
113
|
/** Get dataset metadata (tags, custom fields, checklists). */
|
|
75
114
|
metadata(datasetName: string, projectKey?: string): Promise<Record<string, unknown>>;
|
|
@@ -102,6 +141,10 @@ export declare class DatasetsResource extends BaseResource {
|
|
|
102
141
|
create(opts: DatasetCreateOptions): Promise<Record<string, unknown>>;
|
|
103
142
|
/** Validate common build blockers before running a dataset build. */
|
|
104
143
|
validateBuildSettings(datasetName: string, projectKey?: string): Promise<DatasetBuildValidationResult>;
|
|
144
|
+
/** List files stored by an UploadedFiles dataset. */
|
|
145
|
+
listUploadedFiles(datasetName: string, projectKey?: string): Promise<UploadedFileMetadata[]>;
|
|
146
|
+
/** Add a new file to an UploadedFiles dataset and verify its byte length. */
|
|
147
|
+
uploadDatasetFile(datasetName: string, localPath: string, opts: UploadDatasetFileOptions): Promise<UploadDatasetFileResult>;
|
|
105
148
|
/** Clone dataset settings, preserving connection/storage, format, and schema fields. */
|
|
106
149
|
clone(sourceName: string, targetName: string, opts?: DatasetCloneOptions): Promise<DatasetCloneResult>;
|
|
107
150
|
/** Update a dataset by deep-merging a patch into the current definition. */
|