dataiku-sdk 0.12.7 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -15
- package/bin/dss.js +21 -2
- package/dist/build-metadata.json +1 -0
- package/dist/packages/types/src/index.d.ts +128 -3
- package/dist/packages/types/src/index.js +106 -3
- package/dist/src/cli/coerce.d.ts +2 -1
- package/dist/src/cli/coerce.js +84 -26
- package/dist/src/cli/commands/app.js +6 -6
- package/dist/src/cli/commands/code.js +2 -2
- package/dist/src/cli/commands/data-quality.js +50 -8
- package/dist/src/cli/commands/dataset.d.ts +23 -0
- package/dist/src/cli/commands/dataset.js +197 -12
- package/dist/src/cli/commands/folder.js +5 -4
- package/dist/src/cli/commands/future.js +2 -2
- package/dist/src/cli/commands/index.js +2 -0
- package/dist/src/cli/commands/job.js +26 -13
- package/dist/src/cli/commands/notebook.js +1 -1
- package/dist/src/cli/commands/project-git.d.ts +2 -0
- package/dist/src/cli/commands/project-git.js +563 -0
- package/dist/src/cli/commands/project.js +118 -14
- package/dist/src/cli/commands/recipe.js +14 -17
- package/dist/src/cli/commands/scenario.js +4 -4
- package/dist/src/cli/contract.d.ts +6 -4
- package/dist/src/cli/contract.js +595 -68
- package/dist/src/cli/doctor.d.ts +12 -2
- package/dist/src/cli/doctor.js +50 -19
- package/dist/src/cli/flags.js +23 -2
- package/dist/src/cli/helpers/cleanup.js +30 -1
- package/dist/src/cli/helpers/dataset.js +1 -1
- package/dist/src/cli/output.d.ts +5 -2
- package/dist/src/cli/output.js +13 -12
- package/dist/src/cli/types.d.ts +2 -0
- package/dist/src/cli/usage.d.ts +1 -1
- package/dist/src/cli/usage.js +2 -1
- package/dist/src/cli/version.d.ts +48 -6
- package/dist/src/cli/version.js +177 -11
- package/dist/src/cli.js +94 -39
- package/dist/src/client.d.ts +31 -1
- package/dist/src/client.js +39 -7
- package/dist/src/errors.d.ts +1 -1
- package/dist/src/index.d.ts +6 -4
- package/dist/src/index.js +3 -1
- package/dist/src/resources/connections.d.ts +3 -1
- package/dist/src/resources/connections.js +82 -21
- package/dist/src/resources/datasets.d.ts +43 -0
- package/dist/src/resources/datasets.js +241 -30
- package/dist/src/resources/folders.js +7 -3
- package/dist/src/resources/project-git.d.ts +154 -0
- package/dist/src/resources/project-git.js +427 -0
- package/dist/src/resources/projects.d.ts +36 -3
- package/dist/src/resources/projects.js +127 -16
- package/dist/src/resources/recipes.js +4 -4
- package/dist/src/schemas.d.ts +2 -2
- package/dist/src/schemas.js +1 -1
- package/dist/src/utils/project-archive.d.ts +32 -0
- package/dist/src/utils/project-archive.js +624 -0
- package/package.json +6 -5
- package/packages/types/dist/index.d.ts +128 -3
- package/packages/types/dist/index.js +106 -3
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +26 -19
|
@@ -0,0 +1,427 @@
|
|
|
1
|
+
import { ClientValidationError, DataikuError, } from "../errors.js";
|
|
2
|
+
import { ProjectGitActionResultSchema, ProjectGitDiffResultSchema, ProjectGitFutureResponseSchema, ProjectGitFutureStateSchema, ProjectGitLibrariesSchema, ProjectGitLogResultSchema, ProjectGitRemoteSchema, ProjectGitStatusSchema, ProjectGitTagsSchema, } from "../schemas.js";
|
|
3
|
+
import { BaseResource, } from "./base.js";
|
|
4
|
+
/**
|
|
5
|
+
* The per-project Git routes exist only under the dataikuapi base, not under
|
|
6
|
+
* the documented `/public/api` REST prefix used by the rest of this SDK, so
|
|
7
|
+
* this family carries its own mount point.
|
|
8
|
+
*/
|
|
9
|
+
const GIT_API_BASE = "/dip/publicapi";
|
|
10
|
+
const DEFAULT_REMOTE = "origin";
|
|
11
|
+
const DEFAULT_LOG_COUNT = 1_000;
|
|
12
|
+
const DEFAULT_TAG_REFERENCE = "HEAD";
|
|
13
|
+
const DEFAULT_POLL_INTERVAL_MS = 2_000;
|
|
14
|
+
const DEFAULT_TIMEOUT_MS = 120_000;
|
|
15
|
+
function sleep(ms) {
|
|
16
|
+
const { promise, resolve, } = Promise.withResolvers();
|
|
17
|
+
setTimeout(resolve, ms);
|
|
18
|
+
return promise;
|
|
19
|
+
}
|
|
20
|
+
function encodePathSegments(path) {
|
|
21
|
+
const normalized = path.replace(/^\/+/, "").replace(/\/+$/, "");
|
|
22
|
+
if (!normalized)
|
|
23
|
+
throw new ClientValidationError("A Git reference path is required.");
|
|
24
|
+
return normalized
|
|
25
|
+
.split("/")
|
|
26
|
+
.map((segment) => encodeURIComponent(segment))
|
|
27
|
+
.join("/");
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Reject `https://user:token@host/repo` before it reaches DSS: an embedded
|
|
31
|
+
* secret would be stored in the project's Git configuration and echoed back by
|
|
32
|
+
* later reads. SSH and scp-style remotes (`git@host:org/repo.git`) are the
|
|
33
|
+
* documented way to authenticate and stay allowed.
|
|
34
|
+
*/
|
|
35
|
+
function validatedRepository(repository, field) {
|
|
36
|
+
const candidate = repository.trim();
|
|
37
|
+
if (candidate === "")
|
|
38
|
+
throw new ClientValidationError(`${field} is required.`);
|
|
39
|
+
if (/[\u0000-\u001f\u007f]/u.test(candidate)) {
|
|
40
|
+
throw new ClientValidationError(`${field} must not contain control characters.`);
|
|
41
|
+
}
|
|
42
|
+
if (/^https?:/i.test(candidate)) {
|
|
43
|
+
if (candidate.includes("\\")) {
|
|
44
|
+
throw new ClientValidationError(`${field} must be a valid HTTP(S) URL.`);
|
|
45
|
+
}
|
|
46
|
+
if (!/^https?:\/\//i.test(candidate)) {
|
|
47
|
+
throw new ClientValidationError(`${field} must be a valid HTTP(S) URL.`);
|
|
48
|
+
}
|
|
49
|
+
let parsed;
|
|
50
|
+
try {
|
|
51
|
+
parsed = new URL(candidate);
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
throw new ClientValidationError(`${field} must be a valid HTTP(S) URL.`);
|
|
55
|
+
}
|
|
56
|
+
if (parsed.username !== "" || parsed.password !== "") {
|
|
57
|
+
throw new ClientValidationError(`${field} must not embed credentials (user:password@host). `
|
|
58
|
+
+ "Use an SSH remote or configure the Git credentials in DSS instead.");
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return candidate;
|
|
62
|
+
}
|
|
63
|
+
/** Replace every exact occurrence of a secret so echoed errors cannot leak it. */
|
|
64
|
+
function redactSecrets(text, secrets) {
|
|
65
|
+
let result = text;
|
|
66
|
+
for (const secret of secrets) {
|
|
67
|
+
if (secret !== undefined && secret !== "") {
|
|
68
|
+
result = result.split(secret).join("[redacted]");
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return result;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Strip a password that a server misconfiguration may have echoed from an
|
|
75
|
+
* error response. `DataikuError.message` was built at construction, so the
|
|
76
|
+
* raw body, the message, and the stack all need scrubbing.
|
|
77
|
+
*/
|
|
78
|
+
function scrubErrorSecrets(error, secrets) {
|
|
79
|
+
if (!(error instanceof DataikuError) || secrets.length === 0)
|
|
80
|
+
return;
|
|
81
|
+
error.body = redactSecrets(error.body, secrets);
|
|
82
|
+
error.message = redactSecrets(error.message, secrets);
|
|
83
|
+
if (typeof error.stack === "string") {
|
|
84
|
+
error.stack = redactSecrets(error.stack, secrets);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
function describeFutureFailure(error) {
|
|
88
|
+
if (error === undefined || error === null)
|
|
89
|
+
return undefined;
|
|
90
|
+
if (typeof error === "string")
|
|
91
|
+
return error.trim() === "" ? undefined : error;
|
|
92
|
+
if (typeof error === "object") {
|
|
93
|
+
const record = error;
|
|
94
|
+
for (const key of ["detailedMessage", "message", "errorType",]) {
|
|
95
|
+
const value = record[key];
|
|
96
|
+
if (typeof value === "string" && value.trim() !== "")
|
|
97
|
+
return value;
|
|
98
|
+
}
|
|
99
|
+
return JSON.stringify(error);
|
|
100
|
+
}
|
|
101
|
+
return String(error);
|
|
102
|
+
}
|
|
103
|
+
/* ------------------------------------------------------------------ */
|
|
104
|
+
/* Resource */
|
|
105
|
+
/* ------------------------------------------------------------------ */
|
|
106
|
+
export class ProjectGitResource extends BaseResource {
|
|
107
|
+
/* ---- status & remotes ---- */
|
|
108
|
+
/** Working-copy status: current branch, remotes, tracking counts, changes. */
|
|
109
|
+
async status(projectKey) {
|
|
110
|
+
const raw = await this.request("GET", `${this.gitPath(projectKey)}/status`);
|
|
111
|
+
return this.client.safeParse(ProjectGitStatusSchema, raw ?? {}, "projectGit.status");
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Read a Git remote. DSS answers `{}` when the remote is not configured.
|
|
115
|
+
* The remote name defaults to `origin` like the Python client.
|
|
116
|
+
*/
|
|
117
|
+
async getRemote(projectKey, name = DEFAULT_REMOTE) {
|
|
118
|
+
const raw = await this.request("GET", `${this.gitPath(projectKey)}/remotes/${encodeURIComponent(name)}`);
|
|
119
|
+
return this.client.safeParse(ProjectGitRemoteSchema, raw ?? {}, "projectGit.getRemote");
|
|
120
|
+
}
|
|
121
|
+
/** Create or replace a Git remote. */
|
|
122
|
+
async setRemote(projectKey, options) {
|
|
123
|
+
const repository = validatedRepository(options.url, "Git remote URL");
|
|
124
|
+
const raw = await this.request("POST", `${this.gitPath(projectKey)}/remotes/${encodeURIComponent(options.name ?? DEFAULT_REMOTE)}`, { url: repository, });
|
|
125
|
+
return this.client.safeParse(ProjectGitActionResultSchema, raw ?? {}, "projectGit.setRemote");
|
|
126
|
+
}
|
|
127
|
+
/** Delete a Git remote. */
|
|
128
|
+
async removeRemote(projectKey, name = DEFAULT_REMOTE) {
|
|
129
|
+
await this.request("DELETE", `${this.gitPath(projectKey)}/remotes/${encodeURIComponent(name)}`);
|
|
130
|
+
}
|
|
131
|
+
/* ---- branches ---- */
|
|
132
|
+
/** Plain list of branch names, local by default. */
|
|
133
|
+
async listBranches(projectKey, remote = false) {
|
|
134
|
+
const raw = await this.request("GET", `${this.gitPath(projectKey)}/branches${this.query({ remote, })}`);
|
|
135
|
+
return (raw ?? []);
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Create a branch and switch to it. With `duplicateProject`, DSS creates a
|
|
139
|
+
* duplicate project for the branch instead of moving this one.
|
|
140
|
+
*/
|
|
141
|
+
async createBranch(projectKey, name, options = {}) {
|
|
142
|
+
// DSS consumes the full key set, so absent values travel as nulls.
|
|
143
|
+
const raw = await this.request("POST", `${this.gitPath(projectKey)}/branches/`, {
|
|
144
|
+
name,
|
|
145
|
+
commit: options.commit ?? null,
|
|
146
|
+
duplicateProject: options.duplicateProject ?? false,
|
|
147
|
+
targetProjectKey: options.targetProjectKey ?? null,
|
|
148
|
+
targetProjectFolderId: options.targetProjectFolderId ?? null,
|
|
149
|
+
});
|
|
150
|
+
return this.client.safeParse(ProjectGitActionResultSchema, raw ?? {}, "projectGit.createBranch");
|
|
151
|
+
}
|
|
152
|
+
/** Delete a branch locally and, with `deleteRemotely`, on the remote too. */
|
|
153
|
+
async deleteBranch(projectKey, name, options = {}) {
|
|
154
|
+
const raw = await this.request("POST", `${this.actionPath(projectKey)}/deleteBranch`, {
|
|
155
|
+
name,
|
|
156
|
+
remote: options.remote ?? false,
|
|
157
|
+
deleteRemotely: options.deleteRemotely ?? false,
|
|
158
|
+
forceDelete: options.forceDelete ?? false,
|
|
159
|
+
});
|
|
160
|
+
return this.client.safeParse(ProjectGitActionResultSchema, raw ?? {}, "projectGit.deleteBranch");
|
|
161
|
+
}
|
|
162
|
+
/** Current branch name, or null when DSS reports none. */
|
|
163
|
+
async currentBranch(projectKey) {
|
|
164
|
+
const raw = await this.request("GET", `${this.gitPath(projectKey)}/current-branch`);
|
|
165
|
+
const name = raw?.name;
|
|
166
|
+
return typeof name === "string" && name !== "" ? name : null;
|
|
167
|
+
}
|
|
168
|
+
/** Switch the project to another branch. */
|
|
169
|
+
async switchBranch(projectKey, branchName) {
|
|
170
|
+
const raw = await this.request("POST", `${this.actionPath(projectKey)}/switchBranch${this.query({ branchName, })}`);
|
|
171
|
+
return this.client.safeParse(ProjectGitActionResultSchema, raw ?? {}, "projectGit.switchBranch");
|
|
172
|
+
}
|
|
173
|
+
/* ---- tags ---- */
|
|
174
|
+
async listTags(projectKey) {
|
|
175
|
+
const raw = await this.request("GET", `${this.gitPath(projectKey)}/tags`);
|
|
176
|
+
return this.client.safeParse(ProjectGitTagsSchema, raw ?? [], "projectGit.listTags");
|
|
177
|
+
}
|
|
178
|
+
async createTag(projectKey, name, options = {}) {
|
|
179
|
+
const raw = await this.request("POST", `${this.gitPath(projectKey)}/tags/`, {
|
|
180
|
+
name,
|
|
181
|
+
reference: options.reference ?? DEFAULT_TAG_REFERENCE,
|
|
182
|
+
message: options.message ?? "",
|
|
183
|
+
});
|
|
184
|
+
return this.client.safeParse(ProjectGitActionResultSchema, raw ?? {}, "projectGit.createTag");
|
|
185
|
+
}
|
|
186
|
+
async deleteTag(projectKey, name) {
|
|
187
|
+
const raw = await this.request("POST", `${this.actionPath(projectKey)}/deleteTag`, {
|
|
188
|
+
name,
|
|
189
|
+
});
|
|
190
|
+
return this.client.safeParse(ProjectGitActionResultSchema, raw ?? {}, "projectGit.deleteTag");
|
|
191
|
+
}
|
|
192
|
+
/* ---- synchronous Git actions ---- */
|
|
193
|
+
/** Fetch from the remote. Failures arrive as `{success: false}` with HTTP 200. */
|
|
194
|
+
async fetch(projectKey) {
|
|
195
|
+
const raw = await this.request("POST", `${this.actionPath(projectKey)}/fetch`);
|
|
196
|
+
return this.client.safeParse(ProjectGitActionResultSchema, raw ?? {}, "projectGit.fetch");
|
|
197
|
+
}
|
|
198
|
+
/** Pull. DSS only ever exposes a rebase pull; there is no merge variant. */
|
|
199
|
+
async pull(projectKey, options = {}) {
|
|
200
|
+
const raw = await this.request("POST", `${this.actionPath(projectKey)}/pullRebase${this.query({ branchName: options.branchName, })}`);
|
|
201
|
+
return this.client.safeParse(ProjectGitActionResultSchema, raw ?? {}, "projectGit.pull");
|
|
202
|
+
}
|
|
203
|
+
async push(projectKey, options = {}) {
|
|
204
|
+
const raw = await this.request("POST", `${this.actionPath(projectKey)}/push${this.query({ branchName: options.branchName, })}`);
|
|
205
|
+
return this.client.safeParse(ProjectGitActionResultSchema, raw ?? {}, "projectGit.push");
|
|
206
|
+
}
|
|
207
|
+
/** Commit log page. Paginate by passing the previous `nextCommit` as `startCommit`. */
|
|
208
|
+
async log(projectKey, options = {}) {
|
|
209
|
+
const raw = await this.request("GET", `${this.actionPath(projectKey)}/log${this.query({
|
|
210
|
+
path: options.path,
|
|
211
|
+
startCommit: options.startCommit,
|
|
212
|
+
count: options.count ?? DEFAULT_LOG_COUNT,
|
|
213
|
+
})}`);
|
|
214
|
+
return this.client.safeParse(ProjectGitLogResultSchema, raw ?? { entries: [], }, "projectGit.log");
|
|
215
|
+
}
|
|
216
|
+
async diff(projectKey, options = {}) {
|
|
217
|
+
const raw = await this.request("GET", `${this.actionPath(projectKey)}/diff${this.query({ commitFrom: options.commitFrom, commitTo: options.commitTo, })}`);
|
|
218
|
+
return this.client.safeParse(ProjectGitDiffResultSchema, raw ?? {}, "projectGit.diff");
|
|
219
|
+
}
|
|
220
|
+
/** Commit the project. DSS stages untracked files before committing. */
|
|
221
|
+
async commit(projectKey, options) {
|
|
222
|
+
const raw = await this.request("POST", `${this.actionPath(projectKey)}/commit`, {
|
|
223
|
+
message: options.message,
|
|
224
|
+
});
|
|
225
|
+
return this.client.safeParse(ProjectGitActionResultSchema, raw ?? {}, "projectGit.commit");
|
|
226
|
+
}
|
|
227
|
+
/** Restore the project to the state of a past commit. */
|
|
228
|
+
async revertToRevision(projectKey, commit) {
|
|
229
|
+
const raw = await this.request("POST", `${this.actionPath(projectKey)}/revertToRevision${this.query({ commit, })}`);
|
|
230
|
+
return this.client.safeParse(ProjectGitActionResultSchema, raw ?? {}, "projectGit.revertToRevision");
|
|
231
|
+
}
|
|
232
|
+
/** Revert one commit, keeping later history. */
|
|
233
|
+
async revertCommit(projectKey, commit) {
|
|
234
|
+
const raw = await this.request("POST", `${this.actionPath(projectKey)}/revertCommit${this.query({ commit, })}`);
|
|
235
|
+
return this.client.safeParse(ProjectGitActionResultSchema, raw ?? {}, "projectGit.revertCommit");
|
|
236
|
+
}
|
|
237
|
+
/** Hard reset to local HEAD, dropping uncommitted changes. */
|
|
238
|
+
async resetToHead(projectKey) {
|
|
239
|
+
const raw = await this.request("POST", `${this.actionPath(projectKey)}/resetToLocalHeadState`);
|
|
240
|
+
return this.client.safeParse(ProjectGitActionResultSchema, raw ?? {}, "projectGit.resetToHead");
|
|
241
|
+
}
|
|
242
|
+
/** Hard reset to the tracked upstream branch, dropping local commits. */
|
|
243
|
+
async resetToUpstream(projectKey) {
|
|
244
|
+
const raw = await this.request("POST", `${this.actionPath(projectKey)}/resetToRemoteHeadState`);
|
|
245
|
+
return this.client.safeParse(ProjectGitActionResultSchema, raw ?? {}, "projectGit.resetToUpstream");
|
|
246
|
+
}
|
|
247
|
+
/**
|
|
248
|
+
* Wipe the project's entire Git history and rebuild a fresh repository.
|
|
249
|
+
* Irreversible, so nothing is sent unless the caller acknowledges it.
|
|
250
|
+
*/
|
|
251
|
+
async dropAndRebuild(projectKey, options = {}) {
|
|
252
|
+
if (options.confirmed !== true) {
|
|
253
|
+
throw new ClientValidationError("projectGit.dropAndRebuild permanently destroys the project's Git history. "
|
|
254
|
+
+ "Pass { confirmed: true } to acknowledge before calling it.");
|
|
255
|
+
}
|
|
256
|
+
const raw = await this.request("POST", `${this.actionPath(projectKey)}/dropAndRebuild${this.query({ iKnowWhatIAmDoing: true, })}`);
|
|
257
|
+
return this.client.safeParse(ProjectGitActionResultSchema, raw ?? {}, "projectGit.dropAndRebuild");
|
|
258
|
+
}
|
|
259
|
+
/* ---- external libraries (async futures) ---- */
|
|
260
|
+
async listLibraries(projectKey) {
|
|
261
|
+
const raw = await this.request("GET", `${this.libraryPath(projectKey)}/`);
|
|
262
|
+
return this.client.safeParse(ProjectGitLibrariesSchema, raw ?? [], "projectGit.listLibraries");
|
|
263
|
+
}
|
|
264
|
+
/** Attach an external Git library. Runs as a DSS future. */
|
|
265
|
+
async addLibrary(projectKey, options) {
|
|
266
|
+
const repository = validatedRepository(options.repository, "Git library repository URL");
|
|
267
|
+
const secrets = [options.password,].filter((secret) => secret !== undefined && secret !== "");
|
|
268
|
+
const raw = await this.request("POST", `${this.libraryPath(projectKey)}/`, {
|
|
269
|
+
repository,
|
|
270
|
+
login: options.login ?? null,
|
|
271
|
+
password: options.password ?? null,
|
|
272
|
+
pathInGitRepository: options.pathInGitRepository ?? "",
|
|
273
|
+
localTargetPath: options.localTargetPath,
|
|
274
|
+
checkout: options.checkout,
|
|
275
|
+
addToPythonPath: options.addToPythonPath ?? true,
|
|
276
|
+
}, secrets);
|
|
277
|
+
return this.client.safeParse(ProjectGitFutureResponseSchema, raw ?? {}, "projectGit.addLibrary");
|
|
278
|
+
}
|
|
279
|
+
/** Update an attached library's repository, sub-path, or checkout. */
|
|
280
|
+
async setLibrary(projectKey, gitReferencePath, options) {
|
|
281
|
+
const repository = validatedRepository(options.repository, "Git library repository URL");
|
|
282
|
+
const secrets = [options.password,].filter((secret) => secret !== undefined && secret !== "");
|
|
283
|
+
const raw = await this.request("PUT", `${this.libraryPath(projectKey)}/${encodePathSegments(gitReferencePath)}`, {
|
|
284
|
+
repository,
|
|
285
|
+
login: options.login ?? null,
|
|
286
|
+
password: options.password ?? null,
|
|
287
|
+
pathInGitRepository: options.pathInGitRepository ?? "",
|
|
288
|
+
checkout: options.checkout,
|
|
289
|
+
}, secrets);
|
|
290
|
+
return raw;
|
|
291
|
+
}
|
|
292
|
+
/** Detach a library, optionally deleting its local directory. */
|
|
293
|
+
async removeLibrary(projectKey, gitReferencePath, deleteDirectory = false) {
|
|
294
|
+
await this.request("DELETE", `${this.libraryPath(projectKey)}/${encodePathSegments(gitReferencePath)}${this.query({ deleteDirectory, })}`);
|
|
295
|
+
}
|
|
296
|
+
/** Reset one library to its remote state. Runs as a DSS future. */
|
|
297
|
+
async resetLibrary(projectKey, gitReferencePath) {
|
|
298
|
+
const raw = await this.request("POST", `${this.libraryPath(projectKey)}/action/reset`, {
|
|
299
|
+
gitRef: gitReferencePath,
|
|
300
|
+
});
|
|
301
|
+
return this.client.safeParse(ProjectGitFutureResponseSchema, raw ?? {}, "projectGit.resetLibrary");
|
|
302
|
+
}
|
|
303
|
+
/** Commit and push one library. Runs as a DSS future. */
|
|
304
|
+
async pushLibrary(projectKey, gitReferencePath, commitMessage) {
|
|
305
|
+
const raw = await this.request("POST", `${this.libraryPath(projectKey)}/action/push`, {
|
|
306
|
+
gitRef: gitReferencePath,
|
|
307
|
+
commitMessage,
|
|
308
|
+
});
|
|
309
|
+
return this.client.safeParse(ProjectGitFutureResponseSchema, raw ?? {}, "projectGit.pushLibrary");
|
|
310
|
+
}
|
|
311
|
+
/** Commit and push every attached library. Runs as a DSS future. */
|
|
312
|
+
async pushAllLibraries(projectKey, commitMessage) {
|
|
313
|
+
const raw = await this.request("POST", `${this.actionPath(projectKey)}/git-refs/push-all`, { commitMessage, });
|
|
314
|
+
return this.client.safeParse(ProjectGitFutureResponseSchema, raw ?? {}, "projectGit.pushAllLibraries");
|
|
315
|
+
}
|
|
316
|
+
/** Reset every attached library to its remote state. Runs as a DSS future. */
|
|
317
|
+
async resetAllLibraries(projectKey) {
|
|
318
|
+
const raw = await this.request("POST", `${this.actionPath(projectKey)}/git-refs/reset-all`);
|
|
319
|
+
return this.client.safeParse(ProjectGitFutureResponseSchema, raw ?? {}, "projectGit.resetAllLibraries");
|
|
320
|
+
}
|
|
321
|
+
/* ---- futures ---- */
|
|
322
|
+
/**
|
|
323
|
+
* Read a future's state. `peek` leaves the future's log unconsumed, which is
|
|
324
|
+
* what polling loops want; the default consumes it like the Python client.
|
|
325
|
+
*/
|
|
326
|
+
async getFutureState(jobId, options = {}) {
|
|
327
|
+
const raw = await this.request("GET", `/futures/${encodeURIComponent(jobId)}${this.query({ peek: options.peek === true, })}`);
|
|
328
|
+
return this.client.safeParse(ProjectGitFutureStateSchema, raw ?? {}, "projectGit.getFutureState");
|
|
329
|
+
}
|
|
330
|
+
/** Abort a running future. */
|
|
331
|
+
async abortFuture(jobId) {
|
|
332
|
+
await this.request("DELETE", `/futures/${encodeURIComponent(jobId)}`);
|
|
333
|
+
}
|
|
334
|
+
/**
|
|
335
|
+
* Poll until the future carries a result and return that result. Throws on
|
|
336
|
+
* an aborted or failed future and on an exhausted budget, so a caller never
|
|
337
|
+
* mistakes an unfinished future for a successful one.
|
|
338
|
+
*/
|
|
339
|
+
async waitForFuture(jobId, options = {}) {
|
|
340
|
+
const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
|
|
341
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
342
|
+
if (!Number.isFinite(pollIntervalMs) || pollIntervalMs <= 0) {
|
|
343
|
+
throw new ClientValidationError("pollIntervalMs must be a finite positive number.");
|
|
344
|
+
}
|
|
345
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs < 0) {
|
|
346
|
+
throw new ClientValidationError("timeoutMs must be a finite non-negative number.");
|
|
347
|
+
}
|
|
348
|
+
const startedAt = Date.now();
|
|
349
|
+
while (true) {
|
|
350
|
+
// Unlike peek state, the ordinary state includes the completed result.
|
|
351
|
+
const state = await this.getFutureState(jobId, { peek: false, });
|
|
352
|
+
const failure = describeFutureFailure(state.error);
|
|
353
|
+
if (failure !== undefined) {
|
|
354
|
+
throw new Error(`Dataiku future ${jobId} failed: ${failure}`);
|
|
355
|
+
}
|
|
356
|
+
if (state.aborted === true) {
|
|
357
|
+
throw new Error(`Dataiku future ${jobId} was aborted`);
|
|
358
|
+
}
|
|
359
|
+
if (state.hasResult === true)
|
|
360
|
+
return state.result;
|
|
361
|
+
if (state.unknown === true) {
|
|
362
|
+
throw new Error(`Dataiku future ${jobId} is unknown to the server`);
|
|
363
|
+
}
|
|
364
|
+
if (state.alive === false) {
|
|
365
|
+
throw new Error(`Dataiku future ${jobId} ended without producing a result`);
|
|
366
|
+
}
|
|
367
|
+
// Only the remaining budget is slept, so a caller's deadline is never
|
|
368
|
+
// overshot by a whole poll interval.
|
|
369
|
+
const elapsedMs = Date.now() - startedAt;
|
|
370
|
+
if (elapsedMs >= timeoutMs) {
|
|
371
|
+
throw new Error(`Timed out after ${String(elapsedMs)}ms waiting for Dataiku future ${jobId}`);
|
|
372
|
+
}
|
|
373
|
+
await sleep(Math.min(pollIntervalMs, timeoutMs - elapsedMs));
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
/* ---- private ---- */
|
|
377
|
+
gitPath(projectKey) {
|
|
378
|
+
return `/projects/${this.enc(projectKey)}/git`;
|
|
379
|
+
}
|
|
380
|
+
actionPath(projectKey) {
|
|
381
|
+
return `${this.gitPath(projectKey)}/actions`;
|
|
382
|
+
}
|
|
383
|
+
libraryPath(projectKey) {
|
|
384
|
+
return `${this.gitPath(projectKey)}/lib-git-refs`;
|
|
385
|
+
}
|
|
386
|
+
query(params) {
|
|
387
|
+
const search = new URLSearchParams();
|
|
388
|
+
for (const [key, value,] of Object.entries(params)) {
|
|
389
|
+
if (value === undefined)
|
|
390
|
+
continue;
|
|
391
|
+
search.set(key, String(value));
|
|
392
|
+
}
|
|
393
|
+
const queryString = search.toString();
|
|
394
|
+
return queryString === "" ? "" : `?${queryString}`;
|
|
395
|
+
}
|
|
396
|
+
async request(method, path, body, secrets = []) {
|
|
397
|
+
const transport = this.client;
|
|
398
|
+
const init = {
|
|
399
|
+
method,
|
|
400
|
+
headers: {
|
|
401
|
+
// The Git routes are only reachable with dataikuapi-style Basic auth:
|
|
402
|
+
// API key as the username, empty password.
|
|
403
|
+
Authorization: `Basic ${Buffer.from(`${transport.apiKey}:`, "utf8").toString("base64")}`,
|
|
404
|
+
Accept: "application/json",
|
|
405
|
+
"Content-Type": "application/json",
|
|
406
|
+
},
|
|
407
|
+
};
|
|
408
|
+
if (body !== undefined)
|
|
409
|
+
init.body = JSON.stringify(body);
|
|
410
|
+
try {
|
|
411
|
+
const res = await transport.fetchWithRetry(`${transport.baseUrl}${GIT_API_BASE}${path}`, init);
|
|
412
|
+
const text = await res.text();
|
|
413
|
+
if (text.trim() === "")
|
|
414
|
+
return undefined;
|
|
415
|
+
try {
|
|
416
|
+
return JSON.parse(text);
|
|
417
|
+
}
|
|
418
|
+
catch {
|
|
419
|
+
throw new Error(`Dataiku Git API returned a non-JSON response for ${method} ${path}`);
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
catch (error) {
|
|
423
|
+
scrubErrorSecrets(error, secrets);
|
|
424
|
+
throw error;
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
}
|
|
@@ -64,10 +64,30 @@ export interface ProjectImportUploadResult {
|
|
|
64
64
|
id: string;
|
|
65
65
|
[key: string]: unknown;
|
|
66
66
|
}
|
|
67
|
+
export interface ProjectImportSettings {
|
|
68
|
+
targetProjectKey?: string;
|
|
69
|
+
remapping?: Record<string, unknown>;
|
|
70
|
+
[key: string]: unknown;
|
|
71
|
+
}
|
|
72
|
+
export interface ProjectImportProcessResult {
|
|
73
|
+
/** Present on definitive responses; a missing or non-boolean value makes the outcome indeterminate. */
|
|
74
|
+
success?: boolean;
|
|
75
|
+
/** Project key DSS actually created (after remapping). Definitive success includes it. */
|
|
76
|
+
usedProjectKey?: string;
|
|
77
|
+
[key: string]: unknown;
|
|
78
|
+
}
|
|
79
|
+
export interface ProjectImportResult extends ProjectImportProcessResult {
|
|
80
|
+
importId: string;
|
|
81
|
+
/** Key the import was bound to: the explicit settings target, else the archive manifest's key. Known on success. */
|
|
82
|
+
requestedProjectKey?: string;
|
|
83
|
+
/** True when usedProjectKey differs from requestedProjectKey (set only when both are known). */
|
|
84
|
+
remapped?: boolean;
|
|
85
|
+
/** SHA-256 fingerprint of the landed project's creationTag. Present on verified success. */
|
|
86
|
+
projectIncarnationHash?: string;
|
|
87
|
+
}
|
|
67
88
|
export declare class ProjectsResource extends BaseResource {
|
|
68
89
|
private httpInternals;
|
|
69
90
|
private postStream;
|
|
70
|
-
private uploadJson;
|
|
71
91
|
/** Create a new project. */
|
|
72
92
|
createProject(projectKey: string, name: string, ownerLogin: string, settings?: ProjectLifecycleSettings | null): Promise<Record<string, unknown>>;
|
|
73
93
|
/** Delete a project using the lifecycle endpoint. */
|
|
@@ -76,8 +96,21 @@ export declare class ProjectsResource extends BaseResource {
|
|
|
76
96
|
duplicate(projectKey: string, targetProjectKey: string, targetProjectName: string, options?: ProjectDuplicateOptions): Promise<Record<string, unknown>>;
|
|
77
97
|
/** Export a project archive to a local file. */
|
|
78
98
|
exportArchive(projectKey: string, filePath: string, options?: ProjectExportOptions): Promise<void>;
|
|
79
|
-
/** Upload a project archive and return
|
|
80
|
-
|
|
99
|
+
/** Upload a project archive and return its temporary import handle. */
|
|
100
|
+
prepareProjectImport(filePath: string): Promise<ProjectImportUploadResult>;
|
|
101
|
+
/** Process an uploaded archive into a DSS project. */
|
|
102
|
+
processProjectImport(importId: string, settings?: ProjectImportSettings): Promise<ProjectImportProcessResult>;
|
|
103
|
+
/**
|
|
104
|
+
* Upload and process a project archive.
|
|
105
|
+
*
|
|
106
|
+
* The archive is validated locally before anything is uploaded. Definitive
|
|
107
|
+
* failures surface the process response verbatim (with `importId`); every
|
|
108
|
+
* indeterminate path throws ClientValidationError(`ambiguous_outcome`).
|
|
109
|
+
* Verified success carries the actually-used project key, the requested
|
|
110
|
+
* key when known (with explicit `remapped` reporting), and the landed
|
|
111
|
+
* project's incarnation hash for identity-bound cleanup and guarded deletes.
|
|
112
|
+
*/
|
|
113
|
+
importProjectFromArchive(filePath: string, settings?: ProjectImportSettings): Promise<ProjectImportResult>;
|
|
81
114
|
/** Get project permissions. */
|
|
82
115
|
getPermissions(projectKey?: string): Promise<ProjectPermissions>;
|
|
83
116
|
/** Set project permissions. */
|
|
@@ -1,7 +1,10 @@
|
|
|
1
|
-
import { createWriteStream,
|
|
1
|
+
import { createWriteStream, } from "node:fs";
|
|
2
2
|
import { Writable, } from "node:stream";
|
|
3
|
+
import { ClientValidationError, DataikuError, } from "../errors.js";
|
|
3
4
|
import { ProjectDetailsSchema, ProjectMetadataSchema, ProjectSummaryArraySchema, } from "../schemas.js";
|
|
4
5
|
import { normalizeFlowGraph, } from "../utils/flow-map.js";
|
|
6
|
+
import { assertImportableProjectArchive, } from "../utils/project-archive.js";
|
|
7
|
+
import { projectIncarnationHash, } from "../utils/project-incarnation.js";
|
|
5
8
|
import { BaseResource, } from "./base.js";
|
|
6
9
|
function fetchWithTimeout(label, timeoutMs, fetcher) {
|
|
7
10
|
return new Promise((resolve) => {
|
|
@@ -100,6 +103,7 @@ function truncateFlowMap(normalized, maxNodes, maxEdges) {
|
|
|
100
103
|
const DEFAULT_MAX_NODES = 300;
|
|
101
104
|
const DEFAULT_MAX_EDGES = 600;
|
|
102
105
|
const DEFAULT_METADATA_TIMEOUT_MS = 1_500;
|
|
106
|
+
const PROJECT_IMPORT_AMBIGUOUS_REMEDIATION = "Import processing left DSS state indeterminate. Inspect the archive with `dss project inspect-archive <file>` and the live projects with `dss project list` before retrying the import.";
|
|
103
107
|
export class ProjectsResource extends BaseResource {
|
|
104
108
|
httpInternals() {
|
|
105
109
|
return this.client;
|
|
@@ -116,18 +120,6 @@ export class ProjectsResource extends BaseResource {
|
|
|
116
120
|
body: JSON.stringify(body),
|
|
117
121
|
});
|
|
118
122
|
}
|
|
119
|
-
async uploadJson(path, filePath, fileName) {
|
|
120
|
-
const http = this.httpInternals();
|
|
121
|
-
const fileBlob = await openAsBlob(filePath);
|
|
122
|
-
const formData = new FormData();
|
|
123
|
-
formData.append("file", fileBlob, fileName);
|
|
124
|
-
const res = await http.fetchWithRetry(`${http.baseUrl}${path}`, {
|
|
125
|
-
method: "POST",
|
|
126
|
-
headers: { Authorization: `Bearer ${http.apiKey}`, },
|
|
127
|
-
body: formData,
|
|
128
|
-
});
|
|
129
|
-
return res.json();
|
|
130
|
-
}
|
|
131
123
|
/** Create a new project. */
|
|
132
124
|
async createProject(projectKey, name, ownerLogin, settings) {
|
|
133
125
|
return this.client.post("/public/api/projects/", {
|
|
@@ -176,9 +168,128 @@ export class ProjectsResource extends BaseResource {
|
|
|
176
168
|
throw error;
|
|
177
169
|
}
|
|
178
170
|
}
|
|
179
|
-
/** Upload a project archive and return
|
|
180
|
-
async
|
|
181
|
-
|
|
171
|
+
/** Upload a project archive and return its temporary import handle. */
|
|
172
|
+
async prepareProjectImport(filePath) {
|
|
173
|
+
const upload = await this.client.uploadJson("/public/api/projects/import/upload", filePath, "tmp-import.zip");
|
|
174
|
+
if (!upload || typeof upload.id !== "string" || upload.id.trim() === "") {
|
|
175
|
+
throw new ClientValidationError("DSS accepted the project archive upload but did not return a temporary import id.", "ambiguous_outcome", "The archive may remain in temporary DSS storage, but no project import was started. Inspect DSS before retrying.");
|
|
176
|
+
}
|
|
177
|
+
return upload;
|
|
178
|
+
}
|
|
179
|
+
/** Process an uploaded archive into a DSS project. */
|
|
180
|
+
async processProjectImport(importId, settings = {}) {
|
|
181
|
+
const body = Object.keys(settings).length === 0 ? { _: "_", } : settings;
|
|
182
|
+
try {
|
|
183
|
+
const response = await this.client.post(`/public/api/projects/import/${encodeURIComponent(importId)}/process`, body);
|
|
184
|
+
// An empty or null 2xx body carries no definitive outcome: DSS may
|
|
185
|
+
// have created the project, partially processed it, or done nothing.
|
|
186
|
+
if (!response) {
|
|
187
|
+
throw new ClientValidationError("DSS accepted the import process request but returned no response body: the project may or may not have been created.", "ambiguous_outcome", PROJECT_IMPORT_AMBIGUOUS_REMEDIATION, {
|
|
188
|
+
importId,
|
|
189
|
+
...(typeof settings.targetProjectKey === "string"
|
|
190
|
+
&& settings.targetProjectKey.trim() !== ""
|
|
191
|
+
? { targetProjectKey: settings.targetProjectKey, }
|
|
192
|
+
: {}),
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
return response;
|
|
196
|
+
}
|
|
197
|
+
catch (error) {
|
|
198
|
+
// A transport timeout or server-side 5xx leaves the import outcome
|
|
199
|
+
// unknowable: the project may or may not exist. Surface that as an
|
|
200
|
+
// ambiguous outcome — never as a definitive failure. A 2xx
|
|
201
|
+
// response that carried a non-JSON body has the same property.
|
|
202
|
+
if (error instanceof DataikuError
|
|
203
|
+
&& (error.status === 0
|
|
204
|
+
|| error.status >= 500
|
|
205
|
+
|| (error.status >= 200 && error.status < 300))) {
|
|
206
|
+
throw new ClientValidationError(error.status >= 200 && error.status < 300
|
|
207
|
+
? "DSS returned a non-JSON process response: the import outcome is indeterminate and the project may or may not have been created."
|
|
208
|
+
: "DSS returned no definitive import result: the project may or may not have been created.", "ambiguous_outcome", PROJECT_IMPORT_AMBIGUOUS_REMEDIATION, {
|
|
209
|
+
importId,
|
|
210
|
+
...(typeof settings.targetProjectKey === "string"
|
|
211
|
+
&& settings.targetProjectKey.trim() !== ""
|
|
212
|
+
? { targetProjectKey: settings.targetProjectKey, }
|
|
213
|
+
: {}),
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
throw error;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Upload and process a project archive.
|
|
221
|
+
*
|
|
222
|
+
* The archive is validated locally before anything is uploaded. Definitive
|
|
223
|
+
* failures surface the process response verbatim (with `importId`); every
|
|
224
|
+
* indeterminate path throws ClientValidationError(`ambiguous_outcome`).
|
|
225
|
+
* Verified success carries the actually-used project key, the requested
|
|
226
|
+
* key when known (with explicit `remapped` reporting), and the landed
|
|
227
|
+
* project's incarnation hash for identity-bound cleanup and guarded deletes.
|
|
228
|
+
*/
|
|
229
|
+
async importProjectFromArchive(filePath, settings = {}) {
|
|
230
|
+
// Fatal archive problems abort before any DSS call or upload.
|
|
231
|
+
const inspection = await assertImportableProjectArchive(filePath);
|
|
232
|
+
const requestedProjectKey = typeof settings.targetProjectKey === "string"
|
|
233
|
+
&& settings.targetProjectKey.trim() !== ""
|
|
234
|
+
? settings.targetProjectKey
|
|
235
|
+
: inspection.sourceProjectKey;
|
|
236
|
+
const upload = await this.prepareProjectImport(filePath);
|
|
237
|
+
const result = await this.processProjectImport(upload.id, settings);
|
|
238
|
+
const importId = upload.id;
|
|
239
|
+
if (typeof result.success !== "boolean") {
|
|
240
|
+
throw new ClientValidationError("DSS returned a process response without a boolean success field: the import outcome is indeterminate.", "ambiguous_outcome", PROJECT_IMPORT_AMBIGUOUS_REMEDIATION, {
|
|
241
|
+
importId,
|
|
242
|
+
...(requestedProjectKey !== undefined
|
|
243
|
+
? { targetProjectKey: requestedProjectKey, }
|
|
244
|
+
: {}),
|
|
245
|
+
});
|
|
246
|
+
}
|
|
247
|
+
if (result.success === false) {
|
|
248
|
+
// Definitive DSS-side failure: keep the response authoritative and
|
|
249
|
+
// let the command layer report it as a plain command failure.
|
|
250
|
+
return { ...result, success: false, importId, };
|
|
251
|
+
}
|
|
252
|
+
const usedProjectKey = typeof result.usedProjectKey === "string"
|
|
253
|
+
&& result.usedProjectKey.trim() !== ""
|
|
254
|
+
? result.usedProjectKey
|
|
255
|
+
: undefined;
|
|
256
|
+
if (usedProjectKey === undefined) {
|
|
257
|
+
throw new ClientValidationError("DSS reported import success but did not name the project that was created.", "ambiguous_outcome", PROJECT_IMPORT_AMBIGUOUS_REMEDIATION, {
|
|
258
|
+
importId,
|
|
259
|
+
...(requestedProjectKey !== undefined
|
|
260
|
+
? { targetProjectKey: requestedProjectKey, }
|
|
261
|
+
: {}),
|
|
262
|
+
});
|
|
263
|
+
}
|
|
264
|
+
// Verify the claimed landing and capture identity for cleanup binding.
|
|
265
|
+
let details;
|
|
266
|
+
try {
|
|
267
|
+
details = await this.get(usedProjectKey);
|
|
268
|
+
}
|
|
269
|
+
catch (error) {
|
|
270
|
+
if (error instanceof DataikuError
|
|
271
|
+
&& (error.status === 404 || error.status === 0 || error.status >= 500)) {
|
|
272
|
+
throw new ClientValidationError(`DSS reported import success for project ${usedProjectKey}, but the project could not be re-read for verification.`, "ambiguous_outcome", PROJECT_IMPORT_AMBIGUOUS_REMEDIATION, { importId, usedProjectKey, });
|
|
273
|
+
}
|
|
274
|
+
throw error;
|
|
275
|
+
}
|
|
276
|
+
const incarnation = projectIncarnationHash(usedProjectKey, details);
|
|
277
|
+
if (incarnation === undefined) {
|
|
278
|
+
throw new ClientValidationError(`DSS imported project ${usedProjectKey}, but returned no creationTag identity.`, "ambiguous_outcome", "Refusing to claim identity-bound success without a project-incarnation hash. Verify the project identity before any cleanup or guarded delete.", { importId, usedProjectKey, });
|
|
279
|
+
}
|
|
280
|
+
return {
|
|
281
|
+
...result,
|
|
282
|
+
success: true,
|
|
283
|
+
importId,
|
|
284
|
+
usedProjectKey,
|
|
285
|
+
projectIncarnationHash: incarnation,
|
|
286
|
+
...(requestedProjectKey === undefined
|
|
287
|
+
? {}
|
|
288
|
+
: {
|
|
289
|
+
requestedProjectKey,
|
|
290
|
+
remapped: requestedProjectKey !== usedProjectKey,
|
|
291
|
+
}),
|
|
292
|
+
};
|
|
182
293
|
}
|
|
183
294
|
/** Get project permissions. */
|
|
184
295
|
async getPermissions(projectKey) {
|