dataiku-sdk 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -10
- package/bin/dss.js +67 -78
- package/dist/build-metadata.json +1 -0
- package/dist/src/cli/commands/api-service.js +5 -4
- package/dist/src/cli/commands/app.js +45 -12
- package/dist/src/cli/commands/bundle.js +5 -4
- package/dist/src/cli/commands/data-quality.js +42 -0
- package/dist/src/cli/commands/dataset.d.ts +23 -0
- package/dist/src/cli/commands/dataset.js +187 -6
- package/dist/src/cli/commands/job.js +15 -2
- package/dist/src/cli/commands/project-library.js +4 -4
- package/dist/src/cli/commands/project.js +116 -12
- package/dist/src/cli/contract.d.ts +1 -0
- package/dist/src/cli/contract.js +194 -15
- package/dist/src/cli/doctor.d.ts +12 -2
- package/dist/src/cli/doctor.js +44 -13
- package/dist/src/cli/flags.js +2 -0
- package/dist/src/cli/helpers/app-successor.d.ts +15 -3
- package/dist/src/cli/helpers/app-successor.js +155 -15
- package/dist/src/cli/helpers/cleanup.js +30 -1
- package/dist/src/cli/output.d.ts +3 -1
- package/dist/src/cli/output.js +11 -6
- package/dist/src/cli/types.d.ts +2 -0
- package/dist/src/cli/version.d.ts +47 -4
- package/dist/src/cli/version.js +176 -9
- package/dist/src/cli.js +55 -14
- package/dist/src/client.d.ts +28 -1
- package/dist/src/client.js +34 -7
- package/dist/src/errors.d.ts +1 -1
- package/dist/src/index.d.ts +3 -2
- package/dist/src/index.js +1 -0
- package/dist/src/resources/datasets.d.ts +43 -0
- package/dist/src/resources/datasets.js +199 -28
- package/dist/src/resources/folders.js +6 -8
- package/dist/src/resources/projects.d.ts +36 -3
- package/dist/src/resources/projects.js +122 -20
- package/dist/src/skill.js +1 -18
- package/dist/src/utils/project-archive.d.ts +32 -0
- package/dist/src/utils/project-archive.js +623 -0
- package/dist/src/utils/stable-hash.js +1 -2
- package/package.json +10 -10
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +27 -35
|
@@ -0,0 +1,623 @@
|
|
|
1
|
+
import StreamZip from "node-stream-zip";
|
|
2
|
+
import { stat, } from "node:fs/promises";
|
|
3
|
+
import { ClientValidationError, } from "../errors.js";
|
|
4
|
+
const MANIFEST_MEMBER = "export-manifest.json";
|
|
5
|
+
const DATASETS_ROOT = "project_config/datasets";
|
|
6
|
+
const RECIPES_ROOT = "project_config/recipes";
|
|
7
|
+
const BUNDLED_DATA_ROOT = "any_datasets_data";
|
|
8
|
+
const UPLOADS_ROOT = "uploads";
|
|
9
|
+
/**
|
|
10
|
+
* Definition roots for the non-dataset flow objects a recipe channel can
|
|
11
|
+
* reference. Their names live in the same flat namespace as dataset names, so
|
|
12
|
+
* an untyped reference resolving here is not a dangling dataset reference.
|
|
13
|
+
*/
|
|
14
|
+
const NON_DATASET_OBJECT_ROOTS = [
|
|
15
|
+
"project_config/managed_folders",
|
|
16
|
+
"project_config/saved_models",
|
|
17
|
+
"project_config/streaming_endpoints",
|
|
18
|
+
"project_config/model_evaluation_stores",
|
|
19
|
+
];
|
|
20
|
+
/**
|
|
21
|
+
* Reference `type` values that designate a non-dataset flow object. Refs
|
|
22
|
+
* carrying one of these are validated against their own definition
|
|
23
|
+
* namespace, never against the dataset definitions.
|
|
24
|
+
*/
|
|
25
|
+
const NON_DATASET_REFERENCE_TYPES = {
|
|
26
|
+
MANAGED_FOLDER: true,
|
|
27
|
+
FOLDER: true,
|
|
28
|
+
SAVED_MODEL: true,
|
|
29
|
+
MODEL: true,
|
|
30
|
+
STREAMING_ENDPOINT: true,
|
|
31
|
+
MODEL_EVALUATION_STORE: true,
|
|
32
|
+
};
|
|
33
|
+
/** Upper bound for parsed JSON members (manifest, recipes). */
|
|
34
|
+
const MAX_JSON_MEMBER_BYTES = 16 * 1024 * 1024;
|
|
35
|
+
/** Upper bound for residue text scanning of individual members. */
|
|
36
|
+
const MAX_RESIDUE_SCAN_BYTES = 1024 * 1024;
|
|
37
|
+
function plainRecord(value) {
|
|
38
|
+
return typeof value === "object" && value !== null && !Array.isArray(value)
|
|
39
|
+
? value
|
|
40
|
+
: undefined;
|
|
41
|
+
}
|
|
42
|
+
function stringField(record, key) {
|
|
43
|
+
const value = record[key];
|
|
44
|
+
return typeof value === "string" && value.trim().length > 0 ? value : undefined;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Open a ZIP archive and enumerate every central-directory record via the
|
|
48
|
+
* `entry` event, so duplicate member names stay visible (the `entries()` map
|
|
49
|
+
* collapses them). Name validation is skipped here and performed separately,
|
|
50
|
+
* so unsafe names are reported as issues instead of aborting the inspection.
|
|
51
|
+
*/
|
|
52
|
+
function openArchive(filePath) {
|
|
53
|
+
const { promise, resolve, reject, } = Promise.withResolvers();
|
|
54
|
+
let settled = false;
|
|
55
|
+
const zip = new StreamZip({
|
|
56
|
+
file: filePath,
|
|
57
|
+
storeEntries: true,
|
|
58
|
+
skipEntryNameValidation: true,
|
|
59
|
+
});
|
|
60
|
+
const records = [];
|
|
61
|
+
zip.on("entry", (entry) => {
|
|
62
|
+
records.push(entry);
|
|
63
|
+
});
|
|
64
|
+
zip.on("error", (err) => {
|
|
65
|
+
if (!settled) {
|
|
66
|
+
settled = true;
|
|
67
|
+
try {
|
|
68
|
+
zip.close();
|
|
69
|
+
}
|
|
70
|
+
catch {
|
|
71
|
+
// Best effort: the fd is already held by this invalid handle.
|
|
72
|
+
}
|
|
73
|
+
reject(err);
|
|
74
|
+
}
|
|
75
|
+
});
|
|
76
|
+
zip.on("ready", () => {
|
|
77
|
+
if (!settled) {
|
|
78
|
+
settled = true;
|
|
79
|
+
resolve({ zip, records, });
|
|
80
|
+
}
|
|
81
|
+
});
|
|
82
|
+
return promise;
|
|
83
|
+
}
|
|
84
|
+
function closeArchive(zip) {
|
|
85
|
+
const { promise, resolve, } = Promise.withResolvers();
|
|
86
|
+
zip.close(() => {
|
|
87
|
+
resolve();
|
|
88
|
+
});
|
|
89
|
+
return promise;
|
|
90
|
+
}
|
|
91
|
+
/** Independent member-path safety check (never trusts the library's own). */
|
|
92
|
+
function isUnsafeMemberPath(name) {
|
|
93
|
+
if (name.length === 0 || name.includes("\0"))
|
|
94
|
+
return true;
|
|
95
|
+
if (name.startsWith("/") || name.startsWith("\\"))
|
|
96
|
+
return true;
|
|
97
|
+
if (/^[A-Za-z]:/.test(name))
|
|
98
|
+
return true;
|
|
99
|
+
for (const segment of name.replace(/\\/g, "/").split("/")) {
|
|
100
|
+
if (segment === "..")
|
|
101
|
+
return true;
|
|
102
|
+
}
|
|
103
|
+
return false;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Stream one member through a bounded-memory CRC-32/size check. The library's
|
|
107
|
+
* own verification is bypassed for data-descriptor records (bit 3 set), so the
|
|
108
|
+
* manual `Bun.hash.crc32` computation covers every record uniformly.
|
|
109
|
+
*
|
|
110
|
+
* When `accumulate` is set, buffered content never exceeds `maxBytes`: the
|
|
111
|
+
* buffer is dropped as soon as the cap is reached while the member is still
|
|
112
|
+
* streamed in full, so CRC/size verification stays complete and the size
|
|
113
|
+
* mismatch (if the declared size lied) is still reported.
|
|
114
|
+
*/
|
|
115
|
+
function streamMember(zip, entry, accumulate, maxBytes = Number.POSITIVE_INFINITY) {
|
|
116
|
+
const { promise, resolve, reject, } = Promise.withResolvers();
|
|
117
|
+
zip.stream(entry, (err, stm) => {
|
|
118
|
+
if (err) {
|
|
119
|
+
return reject(err);
|
|
120
|
+
}
|
|
121
|
+
if (!stm) {
|
|
122
|
+
return reject(new Error("archive produced no member stream"));
|
|
123
|
+
}
|
|
124
|
+
const stream = stm;
|
|
125
|
+
let bytes = 0;
|
|
126
|
+
let crc = 0;
|
|
127
|
+
let truncated = false;
|
|
128
|
+
const content = accumulate ? [] : undefined;
|
|
129
|
+
stream.on("data", (chunk) => {
|
|
130
|
+
const buffer = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
|
|
131
|
+
bytes += buffer.length;
|
|
132
|
+
crc = Bun.hash.crc32(buffer, crc) >>> 0;
|
|
133
|
+
if (!content || truncated)
|
|
134
|
+
return;
|
|
135
|
+
if (bytes > maxBytes) {
|
|
136
|
+
truncated = true;
|
|
137
|
+
content.length = 0;
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
content.push(buffer);
|
|
141
|
+
});
|
|
142
|
+
stream.on("end", () => {
|
|
143
|
+
resolve({
|
|
144
|
+
bytes,
|
|
145
|
+
crc,
|
|
146
|
+
content: truncated ? undefined : content,
|
|
147
|
+
truncated,
|
|
148
|
+
});
|
|
149
|
+
});
|
|
150
|
+
stream.on("error", (streamErr) => {
|
|
151
|
+
reject(streamErr);
|
|
152
|
+
});
|
|
153
|
+
});
|
|
154
|
+
return promise;
|
|
155
|
+
}
|
|
156
|
+
function definitionNames(records, root) {
|
|
157
|
+
const names = new Set();
|
|
158
|
+
const prefix = `${root}/`;
|
|
159
|
+
for (const entry of records) {
|
|
160
|
+
const name = entry.name;
|
|
161
|
+
if (!entry.isDirectory && name.startsWith(prefix) && name.endsWith(".json")) {
|
|
162
|
+
names.add(name.slice(prefix.length, -5));
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
return names;
|
|
166
|
+
}
|
|
167
|
+
function datasetDefinitionNames(records) {
|
|
168
|
+
return definitionNames(records, DATASETS_ROOT);
|
|
169
|
+
}
|
|
170
|
+
function recipeDefinitionNames(records) {
|
|
171
|
+
return definitionNames(records, RECIPES_ROOT);
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* Check dangling dataset references in recipe definitions: every `ref` under
|
|
175
|
+
* `inputs`/`outputs` channel items that provably names a local dataset must
|
|
176
|
+
* name a defined dataset. Refs to managed folders, saved models, streaming
|
|
177
|
+
* endpoints or model evaluation stores, and project-qualified (`project.key`)
|
|
178
|
+
* refs are not provable against the local dataset definitions and are never
|
|
179
|
+
* treated as dangling datasets.
|
|
180
|
+
*/
|
|
181
|
+
function checkRecipeReferences(recipes, datasetNames, nonDatasetNames, issues) {
|
|
182
|
+
for (const recipeName of [...recipes.keys(),].sort()) {
|
|
183
|
+
const recipe = plainRecord(recipes.get(recipeName));
|
|
184
|
+
if (!recipe)
|
|
185
|
+
continue;
|
|
186
|
+
for (const direction of ["inputs", "outputs",]) {
|
|
187
|
+
const channels = plainRecord(recipe[direction]);
|
|
188
|
+
if (!channels)
|
|
189
|
+
continue;
|
|
190
|
+
for (const [channelName, channelValue,] of Object.entries(channels)) {
|
|
191
|
+
const channel = plainRecord(channelValue);
|
|
192
|
+
if (!channel)
|
|
193
|
+
continue;
|
|
194
|
+
const items = channel["items"];
|
|
195
|
+
if (!Array.isArray(items))
|
|
196
|
+
continue;
|
|
197
|
+
const channelType = stringField(channel, "type");
|
|
198
|
+
for (const itemValue of items) {
|
|
199
|
+
const item = plainRecord(itemValue) ?? {};
|
|
200
|
+
const ref = stringField(item, "ref");
|
|
201
|
+
if (!ref)
|
|
202
|
+
continue;
|
|
203
|
+
// Project-qualified refs point at the source project's own
|
|
204
|
+
// objects and cannot be proven or disproven locally.
|
|
205
|
+
if (ref.includes("."))
|
|
206
|
+
continue;
|
|
207
|
+
const declaredType = stringField(item, "type")
|
|
208
|
+
?? stringField(item, "objectType")
|
|
209
|
+
?? stringField(item, "kind")
|
|
210
|
+
?? channelType;
|
|
211
|
+
if (declaredType !== undefined) {
|
|
212
|
+
const normalized = declaredType.trim().toUpperCase();
|
|
213
|
+
if (NON_DATASET_REFERENCE_TYPES[normalized] === true)
|
|
214
|
+
continue;
|
|
215
|
+
}
|
|
216
|
+
if (datasetNames.has(ref) || nonDatasetNames.has(ref))
|
|
217
|
+
continue;
|
|
218
|
+
issues.push({
|
|
219
|
+
severity: "error",
|
|
220
|
+
code: "dataset_reference_unresolved",
|
|
221
|
+
message: `recipe ${recipeName} has dangling dataset reference ${direction}.${channelName}.${ref}`,
|
|
222
|
+
member: `${RECIPES_ROOT}/${recipeName}.json`,
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
/**
|
|
230
|
+
* Check orphaned bundled-data members and recipe payload members: every
|
|
231
|
+
* `any_datasets_data/`/`uploads/` root must correspond to a dataset
|
|
232
|
+
* definition or a manifest entry, and every non-`.json` recipe member must
|
|
233
|
+
* belong to a defined recipe.
|
|
234
|
+
*/
|
|
235
|
+
function checkOrphanMembers(records, recipeNames, anchoredDatasetNames, issues) {
|
|
236
|
+
for (const root of [BUNDLED_DATA_ROOT, UPLOADS_ROOT,]) {
|
|
237
|
+
const prefix = `${root}/`;
|
|
238
|
+
const orphans = new Set();
|
|
239
|
+
for (const entry of records) {
|
|
240
|
+
const name = entry.name;
|
|
241
|
+
if (!name.startsWith(prefix))
|
|
242
|
+
continue;
|
|
243
|
+
const datasetName = name.slice(prefix.length).split("/", 1)[0];
|
|
244
|
+
if (datasetName.length > 0 && !anchoredDatasetNames.has(datasetName)) {
|
|
245
|
+
orphans.add(`${root}/${datasetName}`);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
for (const path of [...orphans,].sort()) {
|
|
249
|
+
issues.push({
|
|
250
|
+
severity: "error",
|
|
251
|
+
code: "orphan_member",
|
|
252
|
+
message: `archive member ${path} has no dataset definition or manifest entry`,
|
|
253
|
+
member: path,
|
|
254
|
+
});
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
const orphanPayloads = new Set();
|
|
258
|
+
const prefix = `${RECIPES_ROOT}/`;
|
|
259
|
+
for (const entry of records) {
|
|
260
|
+
const name = entry.name;
|
|
261
|
+
if (entry.isDirectory || !name.startsWith(prefix) || name.endsWith(".json"))
|
|
262
|
+
continue;
|
|
263
|
+
const relative = name.slice(prefix.length);
|
|
264
|
+
const dotIndex = relative.lastIndexOf(".");
|
|
265
|
+
if (dotIndex <= 0)
|
|
266
|
+
continue;
|
|
267
|
+
const recipeName = relative.slice(0, dotIndex);
|
|
268
|
+
if (!recipeNames.has(recipeName))
|
|
269
|
+
orphanPayloads.add(relative);
|
|
270
|
+
}
|
|
271
|
+
for (const relative of [...orphanPayloads,].sort()) {
|
|
272
|
+
issues.push({
|
|
273
|
+
severity: "error",
|
|
274
|
+
code: "orphan_member",
|
|
275
|
+
message: `archive member ${prefix}${relative} has no corresponding recipe definition`,
|
|
276
|
+
member: `${prefix}${relative}`,
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
const CREDENTIAL_VALUE_RE = /\b(?:bearer\s+[A-Za-z0-9._~+/=_-]{8,})\b|(?:authorization|api[-_]?key|apikey|x-api-key)["'\s:=]+[A-Za-z0-9._~+/=_-]{16,}/gi;
|
|
281
|
+
const IDENTITY_EMAIL_RE = /(?<![:/?#&=])[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}(?![/])/g;
|
|
282
|
+
const POSIX_HOME_PATH_RE = /\/(?:Users|home)\/[^/\s"'<>,;)]+/g;
|
|
283
|
+
const WINDOWS_HOME_PATH_RE = /[A-Za-z]:\\Users\\[^\\\s"'<>,;)]+/g;
|
|
284
|
+
const IDENTITY_METADATA_KEY_RE = /\b(?:createdBy|lastModifiedBy|modifiedBy|author|creator|owner|username|ownerEmail|userEmail|email|login)\b\s*:/gi;
|
|
285
|
+
function countResidue(text) {
|
|
286
|
+
return {
|
|
287
|
+
credential: (text.match(CREDENTIAL_VALUE_RE) ?? []).length,
|
|
288
|
+
email: (text.match(IDENTITY_EMAIL_RE) ?? []).length,
|
|
289
|
+
homePath: (text.match(POSIX_HOME_PATH_RE) ?? []).length
|
|
290
|
+
+ (text.match(WINDOWS_HOME_PATH_RE) ?? []).length,
|
|
291
|
+
metadataField: (text.match(IDENTITY_METADATA_KEY_RE) ?? []).length,
|
|
292
|
+
};
|
|
293
|
+
}
|
|
294
|
+
function residueIssue(member, counts) {
|
|
295
|
+
const total = counts.credential + counts.email + counts.homePath + counts.metadataField;
|
|
296
|
+
return {
|
|
297
|
+
severity: "warning",
|
|
298
|
+
code: "identity_residue",
|
|
299
|
+
message: `member contains ${total} identity/history/machine-specific residue pattern(s)`
|
|
300
|
+
+ ` (${counts.credential} credential-like, ${counts.email} email-like,`
|
|
301
|
+
+ ` ${counts.homePath} local-path-like, ${counts.metadataField} identity metadata field-like)`,
|
|
302
|
+
member,
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
/**
|
|
306
|
+
* Inspect a DSS project export ZIP archive without extracting anything to
|
|
307
|
+
* disk: enumerate every central-directory record, validate member paths
|
|
308
|
+
* independently, stream every file member through a size/CRC-32 check with
|
|
309
|
+
* bounded memory (manual `Bun.hash.crc32`, covering data-descriptor
|
|
310
|
+
* records), parse `export-manifest.json` and the source project key, and run
|
|
311
|
+
* the generic dangling-reference/orphan-member graph checks. Identity,
|
|
312
|
+
* history and machine-specific residue is reported only as warnings carrying
|
|
313
|
+
* member paths and pattern counts, never matched values.
|
|
314
|
+
*/
|
|
315
|
+
export async function inspectProjectArchive(filePath) {
|
|
316
|
+
let sizeBytes = 0;
|
|
317
|
+
let statError;
|
|
318
|
+
try {
|
|
319
|
+
const info = await stat(filePath);
|
|
320
|
+
if (!info.isFile()) {
|
|
321
|
+
statError = {
|
|
322
|
+
severity: "error",
|
|
323
|
+
code: "archive_not_regular_file",
|
|
324
|
+
message: "archive path is not a regular file",
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
else if (info.size === 0) {
|
|
328
|
+
statError = {
|
|
329
|
+
severity: "error",
|
|
330
|
+
code: "archive_empty",
|
|
331
|
+
message: "archive file is empty",
|
|
332
|
+
};
|
|
333
|
+
}
|
|
334
|
+
sizeBytes = info.size;
|
|
335
|
+
}
|
|
336
|
+
catch {
|
|
337
|
+
statError = {
|
|
338
|
+
severity: "error",
|
|
339
|
+
code: "archive_not_regular_file",
|
|
340
|
+
message: "archive path does not exist or cannot be read",
|
|
341
|
+
};
|
|
342
|
+
}
|
|
343
|
+
const issues = [];
|
|
344
|
+
if (statError) {
|
|
345
|
+
issues.push(statError);
|
|
346
|
+
return {
|
|
347
|
+
filePath,
|
|
348
|
+
sizeBytes,
|
|
349
|
+
memberCount: 0,
|
|
350
|
+
valid: false,
|
|
351
|
+
issues,
|
|
352
|
+
};
|
|
353
|
+
}
|
|
354
|
+
let zip;
|
|
355
|
+
let records = [];
|
|
356
|
+
try {
|
|
357
|
+
const opened = await openArchive(filePath);
|
|
358
|
+
zip = opened.zip;
|
|
359
|
+
records = opened.records;
|
|
360
|
+
}
|
|
361
|
+
catch {
|
|
362
|
+
issues.push({
|
|
363
|
+
severity: "error",
|
|
364
|
+
code: "archive_invalid_zip",
|
|
365
|
+
message: "archive is not a valid ZIP archive",
|
|
366
|
+
});
|
|
367
|
+
return {
|
|
368
|
+
filePath,
|
|
369
|
+
sizeBytes,
|
|
370
|
+
memberCount: 0,
|
|
371
|
+
valid: false,
|
|
372
|
+
issues,
|
|
373
|
+
};
|
|
374
|
+
}
|
|
375
|
+
try {
|
|
376
|
+
const memberCounts = new Map();
|
|
377
|
+
for (const entry of records) {
|
|
378
|
+
memberCounts.set(entry.name, (memberCounts.get(entry.name) ?? 0) + 1);
|
|
379
|
+
if (isUnsafeMemberPath(entry.name)) {
|
|
380
|
+
issues.push({
|
|
381
|
+
severity: "error",
|
|
382
|
+
code: "unsafe_member_path",
|
|
383
|
+
message: "member path is not a safe relative path",
|
|
384
|
+
member: entry.name,
|
|
385
|
+
});
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
for (const name of [...memberCounts.keys(),].sort()) {
|
|
389
|
+
const count = memberCounts.get(name) ?? 0;
|
|
390
|
+
if (count > 1) {
|
|
391
|
+
issues.push({
|
|
392
|
+
severity: "error",
|
|
393
|
+
code: "duplicate_member_name",
|
|
394
|
+
message: `archive member name appears ${count} times in the central directory`,
|
|
395
|
+
member: name,
|
|
396
|
+
});
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
const manifestJson = new Map();
|
|
400
|
+
const recipes = new Map();
|
|
401
|
+
for (const entry of records) {
|
|
402
|
+
if (!entry.isFile)
|
|
403
|
+
continue;
|
|
404
|
+
const name = entry.name;
|
|
405
|
+
const isManifest = name === MANIFEST_MEMBER;
|
|
406
|
+
const isRecipeJson = name.startsWith(`${RECIPES_ROOT}/`) && name.endsWith(".json");
|
|
407
|
+
if (isManifest && entry.size > MAX_JSON_MEMBER_BYTES) {
|
|
408
|
+
issues.push({
|
|
409
|
+
severity: "error",
|
|
410
|
+
code: "manifest_too_large",
|
|
411
|
+
message: "export-manifest.json member is too large to inspect",
|
|
412
|
+
member: name,
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
|
+
if (isRecipeJson && entry.size > MAX_JSON_MEMBER_BYTES) {
|
|
416
|
+
issues.push({
|
|
417
|
+
severity: "warning",
|
|
418
|
+
code: "member_json_too_large",
|
|
419
|
+
message: "recipe definition member is too large to inspect for references",
|
|
420
|
+
member: name,
|
|
421
|
+
});
|
|
422
|
+
}
|
|
423
|
+
const parseCandidate = (isManifest || isRecipeJson) && entry.size <= MAX_JSON_MEMBER_BYTES;
|
|
424
|
+
const scanCandidate = (isManifest
|
|
425
|
+
|| name.startsWith("project_config/")
|
|
426
|
+
|| name.startsWith(`${BUNDLED_DATA_ROOT}/`)
|
|
427
|
+
|| name.startsWith(`${UPLOADS_ROOT}/`))
|
|
428
|
+
&& entry.size <= MAX_RESIDUE_SCAN_BYTES;
|
|
429
|
+
let streamed;
|
|
430
|
+
try {
|
|
431
|
+
// The buffering cap applies to actual streamed bytes, not the
|
|
432
|
+
// declared size: a member can declare a small size and still
|
|
433
|
+
// inflate arbitrarily, which must not be buffered unbounded.
|
|
434
|
+
streamed = await streamMember(zip, entry, parseCandidate || scanCandidate, parseCandidate
|
|
435
|
+
? MAX_JSON_MEMBER_BYTES
|
|
436
|
+
: scanCandidate
|
|
437
|
+
? MAX_RESIDUE_SCAN_BYTES
|
|
438
|
+
: Number.POSITIVE_INFINITY);
|
|
439
|
+
}
|
|
440
|
+
catch (streamErr) {
|
|
441
|
+
streamed = undefined;
|
|
442
|
+
const prefix = `member ${name}: `;
|
|
443
|
+
if (streamErr instanceof Error && /CRC/i.test(streamErr.message)) {
|
|
444
|
+
issues.push({
|
|
445
|
+
severity: "error",
|
|
446
|
+
code: "member_crc_mismatch",
|
|
447
|
+
message: `${prefix}data CRC-32 does not match its central-directory record`,
|
|
448
|
+
member: name,
|
|
449
|
+
});
|
|
450
|
+
}
|
|
451
|
+
else if (streamErr instanceof Error && /Invalid size/i.test(streamErr.message)) {
|
|
452
|
+
issues.push({
|
|
453
|
+
severity: "error",
|
|
454
|
+
code: "member_size_mismatch",
|
|
455
|
+
message: `${prefix}data length does not match its central-directory record`,
|
|
456
|
+
member: name,
|
|
457
|
+
});
|
|
458
|
+
}
|
|
459
|
+
else {
|
|
460
|
+
issues.push({
|
|
461
|
+
severity: "error",
|
|
462
|
+
code: "member_unreadable",
|
|
463
|
+
message: `${prefix}data cannot be read from archive`,
|
|
464
|
+
member: name,
|
|
465
|
+
});
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
if (!streamed)
|
|
469
|
+
continue;
|
|
470
|
+
if (streamed.bytes !== entry.size) {
|
|
471
|
+
issues.push({
|
|
472
|
+
severity: "error",
|
|
473
|
+
code: "member_size_mismatch",
|
|
474
|
+
message: `member ${name}: data length does not match its central-directory record`,
|
|
475
|
+
member: name,
|
|
476
|
+
});
|
|
477
|
+
}
|
|
478
|
+
if (streamed.crc !== entry.crc) {
|
|
479
|
+
issues.push({
|
|
480
|
+
severity: "error",
|
|
481
|
+
code: "member_crc_mismatch",
|
|
482
|
+
message: `member ${name}: data CRC-32 does not match its central-directory record`,
|
|
483
|
+
member: name,
|
|
484
|
+
});
|
|
485
|
+
}
|
|
486
|
+
if (streamed.truncated) {
|
|
487
|
+
issues.push({
|
|
488
|
+
severity: isManifest ? "error" : "warning",
|
|
489
|
+
code: "member_size_limit_exceeded",
|
|
490
|
+
message: `member ${name}: data exceeds the inspection byte limit and was not buffered for parsing`,
|
|
491
|
+
member: name,
|
|
492
|
+
});
|
|
493
|
+
}
|
|
494
|
+
const content = streamed.content;
|
|
495
|
+
if (!content)
|
|
496
|
+
continue;
|
|
497
|
+
const buffer = Buffer.concat(content);
|
|
498
|
+
if (parseCandidate && isManifest) {
|
|
499
|
+
try {
|
|
500
|
+
manifestJson.set(name, JSON.parse(buffer.toString("utf8")));
|
|
501
|
+
}
|
|
502
|
+
catch {
|
|
503
|
+
issues.push({
|
|
504
|
+
severity: "error",
|
|
505
|
+
code: "manifest_invalid_json",
|
|
506
|
+
message: "export-manifest.json is not valid JSON",
|
|
507
|
+
member: name,
|
|
508
|
+
});
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
if (parseCandidate && isRecipeJson) {
|
|
512
|
+
try {
|
|
513
|
+
recipes.set(name.slice(RECIPES_ROOT.length + 1, -5), JSON.parse(buffer.toString("utf8")));
|
|
514
|
+
}
|
|
515
|
+
catch {
|
|
516
|
+
// Recipe graph is not provable from this member; the ported
|
|
517
|
+
// reference check skips unparseable recipes silently.
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
if (scanCandidate && buffer.length <= MAX_RESIDUE_SCAN_BYTES) {
|
|
521
|
+
const counts = countResidue(buffer.toString("utf8"));
|
|
522
|
+
if (counts.credential + counts.email + counts.homePath + counts.metadataField > 0) {
|
|
523
|
+
issues.push(residueIssue(name, counts));
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
let sourceProjectKey;
|
|
528
|
+
let manifestDatasetNames;
|
|
529
|
+
const manifestMember = manifestJson.get(MANIFEST_MEMBER);
|
|
530
|
+
const manifest = plainRecord(manifestMember);
|
|
531
|
+
if (manifestMember === undefined) {
|
|
532
|
+
issues.push({
|
|
533
|
+
severity: "error",
|
|
534
|
+
code: "manifest_missing",
|
|
535
|
+
message: "archive does not contain export-manifest.json",
|
|
536
|
+
});
|
|
537
|
+
}
|
|
538
|
+
else if (!manifest) {
|
|
539
|
+
issues.push({
|
|
540
|
+
severity: "error",
|
|
541
|
+
code: "manifest_invalid_json",
|
|
542
|
+
message: "export-manifest.json is not a valid JSON object",
|
|
543
|
+
});
|
|
544
|
+
}
|
|
545
|
+
else {
|
|
546
|
+
sourceProjectKey = stringField(manifest, "originalProjectKey")
|
|
547
|
+
?? stringField(manifest, "projectKey");
|
|
548
|
+
if (!sourceProjectKey) {
|
|
549
|
+
issues.push({
|
|
550
|
+
severity: "error",
|
|
551
|
+
code: "manifest_project_key_missing",
|
|
552
|
+
message: "export-manifest.json does not declare a source project key",
|
|
553
|
+
member: MANIFEST_MEMBER,
|
|
554
|
+
});
|
|
555
|
+
}
|
|
556
|
+
const actualContent = plainRecord(manifest["actualContent"]);
|
|
557
|
+
const included = actualContent?.["includedDatasets"];
|
|
558
|
+
if (Array.isArray(included)) {
|
|
559
|
+
const names = new Set();
|
|
560
|
+
for (const itemValue of included) {
|
|
561
|
+
const item = plainRecord(itemValue);
|
|
562
|
+
const name = stringField(item ?? {}, "name");
|
|
563
|
+
if (name)
|
|
564
|
+
names.add(name);
|
|
565
|
+
}
|
|
566
|
+
manifestDatasetNames = names;
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
const datasetNames = datasetDefinitionNames(records);
|
|
570
|
+
const nonDatasetNames = new Set();
|
|
571
|
+
for (const root of NON_DATASET_OBJECT_ROOTS) {
|
|
572
|
+
for (const name of definitionNames(records, root)) {
|
|
573
|
+
nonDatasetNames.add(name);
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
const anchored = new Set(datasetNames);
|
|
577
|
+
for (const name of manifestDatasetNames ?? []) {
|
|
578
|
+
anchored.add(name);
|
|
579
|
+
}
|
|
580
|
+
const recipeNames = recipeDefinitionNames(records);
|
|
581
|
+
checkRecipeReferences(recipes, datasetNames, nonDatasetNames, issues);
|
|
582
|
+
checkOrphanMembers(records, recipeNames, anchored, issues);
|
|
583
|
+
issues.sort((a, b) => {
|
|
584
|
+
const severityOrder = a.severity === b.severity ? 0 : a.severity === "error" ? -1 : 1;
|
|
585
|
+
return (severityOrder
|
|
586
|
+
|| a.code.localeCompare(b.code)
|
|
587
|
+
|| (a.member ?? "").localeCompare(b.member ?? "")
|
|
588
|
+
|| a.message.localeCompare(b.message));
|
|
589
|
+
});
|
|
590
|
+
return {
|
|
591
|
+
filePath,
|
|
592
|
+
sizeBytes,
|
|
593
|
+
memberCount: records.length,
|
|
594
|
+
sourceProjectKey,
|
|
595
|
+
valid: !issues.some((issue) => issue.severity === "error"),
|
|
596
|
+
issues,
|
|
597
|
+
};
|
|
598
|
+
}
|
|
599
|
+
finally {
|
|
600
|
+
if (zip) {
|
|
601
|
+
try {
|
|
602
|
+
await closeArchive(zip);
|
|
603
|
+
}
|
|
604
|
+
catch {
|
|
605
|
+
// The archive is already unreadable; the fd is released on best
|
|
606
|
+
// effort only. Errors here do not change the inspection result.
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
/**
|
|
612
|
+
* Like `inspectProjectArchive`, but throws a `ClientValidationError` with the
|
|
613
|
+
* `validation_failed` code when the inspection contains any error-severity
|
|
614
|
+
* issue, so the archive cannot be imported.
|
|
615
|
+
*/
|
|
616
|
+
export async function assertImportableProjectArchive(filePath) {
|
|
617
|
+
const inspection = await inspectProjectArchive(filePath);
|
|
618
|
+
const errorCount = inspection.issues.filter((issue) => issue.severity === "error").length;
|
|
619
|
+
if (errorCount > 0) {
|
|
620
|
+
throw new ClientValidationError(`project archive is not importable: ${errorCount} validation error(s) found`, "validation_failed", undefined, { filePath: inspection.filePath, issues: inspection.issues, });
|
|
621
|
+
}
|
|
622
|
+
return inspection;
|
|
623
|
+
}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
import { createHash, } from "node:crypto";
|
|
2
1
|
export function sha256Hex(value) {
|
|
3
|
-
return
|
|
2
|
+
return new Bun.CryptoHasher("sha256").update(value).digest("hex");
|
|
4
3
|
}
|
|
5
4
|
/** Locale-independent UTF-16 code-unit order for deterministic machine output. */
|
|
6
5
|
export function compareStrings(a, b) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dataiku-sdk",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.0.0",
|
|
4
4
|
"description": "Dataiku DSS SDK and CLI for programmatic access to DSS REST APIs",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"workspaces": [
|
|
@@ -19,15 +19,14 @@
|
|
|
19
19
|
"packages/types/dist/"
|
|
20
20
|
],
|
|
21
21
|
"engines": {
|
|
22
|
-
"
|
|
23
|
-
"bun": ">=1.3.14"
|
|
22
|
+
"bun": ">=1.4.0"
|
|
24
23
|
},
|
|
25
24
|
"scripts": {
|
|
26
|
-
"build": "tsc && tsc -p packages/types/tsconfig.json",
|
|
27
|
-
"prepack": "
|
|
28
|
-
"prepublishOnly": "
|
|
29
|
-
"check": "tsc --noEmit &&
|
|
30
|
-
"version": "
|
|
25
|
+
"build": "tsc && bun scripts/generate-build-metadata.mjs && tsc -p packages/types/tsconfig.json",
|
|
26
|
+
"prepack": "bun scripts/sync-plugin-version.mjs --check && tsc && bun scripts/generate-build-metadata.mjs && tsc -p packages/types/tsconfig.json",
|
|
27
|
+
"prepublishOnly": "bun scripts/sync-plugin-version.mjs --check && tsc && bun scripts/generate-build-metadata.mjs && tsc -p packages/types/tsconfig.json",
|
|
28
|
+
"check": "tsc --noEmit && bun scripts/sync-plugin-version.mjs --check",
|
|
29
|
+
"version": "bun scripts/sync-plugin-version.mjs && git add plugin.json",
|
|
31
30
|
"lint": "oxlint src/",
|
|
32
31
|
"lint:fix": "oxlint --fix src/",
|
|
33
32
|
"format": "dprint fmt",
|
|
@@ -41,7 +40,7 @@
|
|
|
41
40
|
"test:integration:rigorous:mutating": "cross-env RUN_DATAIKU_INTEGRATION=1 RUN_DATAIKU_INTEGRATION_MUTATING=1 bun test tests/integration-rigorous.test.ts"
|
|
42
41
|
},
|
|
43
42
|
"devDependencies": {
|
|
44
|
-
"@types/bun": "^1.
|
|
43
|
+
"@types/bun": "^1.4.0",
|
|
45
44
|
"cross-env": "^10.1.0",
|
|
46
45
|
"dprint": "^0.55.2",
|
|
47
46
|
"js-tiktoken": "1.0.21",
|
|
@@ -51,7 +50,8 @@
|
|
|
51
50
|
"typescript-language-service": "npm:typescript@5.9.3"
|
|
52
51
|
},
|
|
53
52
|
"dependencies": {
|
|
54
|
-
"@sinclair/typebox": "^0.34.52"
|
|
53
|
+
"@sinclair/typebox": "^0.34.52",
|
|
54
|
+
"node-stream-zip": "^1.16.0"
|
|
55
55
|
},
|
|
56
56
|
"bundledDependencies": [
|
|
57
57
|
"@sinclair/typebox"
|
package/plugin.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json",
|
|
3
3
|
"name": "dataiku-sdk",
|
|
4
|
-
"version": "
|
|
4
|
+
"version": "2.0.0",
|
|
5
5
|
"description": "Dataiku DSS SDK and agent-oriented CLI for programmatic access to DSS REST APIs.",
|
|
6
6
|
"homepage": "https://github.com/clssck/dataiku-sdk",
|
|
7
7
|
"repository": "https://github.com/clssck/dataiku-sdk",
|