dataiku-sdk 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/packages/types/src/index.d.ts +4 -0
- package/dist/packages/types/src/index.js +1 -0
- package/dist/src/cli.js +400 -41
- package/dist/src/errors.js +12 -0
- package/dist/src/index.d.ts +4 -4
- package/dist/src/resources/connections.d.ts +10 -0
- package/dist/src/resources/connections.js +16 -0
- package/dist/src/resources/datasets.d.ts +18 -0
- package/dist/src/resources/datasets.js +41 -0
- package/dist/src/resources/jobs.d.ts +52 -19
- package/dist/src/resources/jobs.js +121 -33
- package/dist/src/resources/notebooks.d.ts +6 -4
- package/dist/src/resources/notebooks.js +6 -4
- package/dist/src/resources/recipes.d.ts +54 -1
- package/dist/src/resources/recipes.js +235 -0
- package/dist/src/resources/sql.js +84 -3
- package/package.json +1 -1
- package/packages/types/dist/index.d.ts +4 -0
- package/packages/types/dist/index.js +1 -0
package/dist/src/errors.js
CHANGED
|
@@ -71,6 +71,18 @@ export function classifyDataikuError(status, body) {
|
|
|
71
71
|
retryHint: "Requested object was not found. Verify projectKey and object identifiers before retrying.",
|
|
72
72
|
};
|
|
73
73
|
}
|
|
74
|
+
const isSqlEngineValidation = status >= 400
|
|
75
|
+
&& (lowerBody.includes("column_not_found")
|
|
76
|
+
|| lowerBody.includes("table_not_found")
|
|
77
|
+
|| lowerBody.includes("no_such_table")
|
|
78
|
+
|| lowerBody.includes("column does not exist"));
|
|
79
|
+
if (isSqlEngineValidation) {
|
|
80
|
+
return {
|
|
81
|
+
category: "validation",
|
|
82
|
+
retryable: false,
|
|
83
|
+
retryHint: "Athena/SQL engine rejected the query: check column names, table names, and schema with dss dataset schema or dss connection tables. Do not retry unchanged SQL.",
|
|
84
|
+
};
|
|
85
|
+
}
|
|
74
86
|
const isServerValidationLike = status >= 500
|
|
75
87
|
&& (lowerBody.includes("invalid")
|
|
76
88
|
|| lowerBody.includes("validation")
|
package/dist/src/index.d.ts
CHANGED
|
@@ -3,18 +3,18 @@ export { type CredentialValidationOptions, type CredentialValidationResult, vali
|
|
|
3
3
|
export { deleteCredentials, type DssCredentials, getConfigDir, getCredentialsPath, loadCredentials, maskApiKey, saveCredentials, } from "./config.js";
|
|
4
4
|
export { DataikuError, type DataikuErrorCategory, type DataikuErrorTaxonomy, type DataikuRetryMetadata, } from "./errors.js";
|
|
5
5
|
export { CodeEnvsResource, } from "./resources/code-envs.js";
|
|
6
|
-
export { ConnectionsResource, } from "./resources/connections.js";
|
|
6
|
+
export { type ConnectionSchemaListOptions, ConnectionsResource, type ConnectionTableListOptions, } from "./resources/connections.js";
|
|
7
7
|
export { DashboardsResource, } from "./resources/dashboards.js";
|
|
8
8
|
export { DataQualityResource, } from "./resources/data-quality.js";
|
|
9
|
-
export { DatasetsResource, } from "./resources/datasets.js";
|
|
9
|
+
export { type DatasetBuildValidationResult, type DatasetSchemaColumnInput, DatasetsResource, } from "./resources/datasets.js";
|
|
10
10
|
export { type FlowZoneItemInput, FlowZonesResource, } from "./resources/flow-zones.js";
|
|
11
11
|
export { FoldersResource, } from "./resources/folders.js";
|
|
12
12
|
export { FuturesResource, } from "./resources/futures.js";
|
|
13
13
|
export { InsightsResource, } from "./resources/insights.js";
|
|
14
|
-
export { computeNextPollDelayMs, type JobBuildTargetType, JobsResource, } from "./resources/jobs.js";
|
|
14
|
+
export { computeNextPollDelayMs, type JobBuildAndWaitOptions, type JobBuildOptions, type JobBuildTarget, type JobBuildTargetType, type JobLogFilter, type JobLogSummary, JobsResource, } from "./resources/jobs.js";
|
|
15
15
|
export { NotebooksResource, } from "./resources/notebooks.js";
|
|
16
16
|
export { type FlowMapResult, ProjectsResource, } from "./resources/projects.js";
|
|
17
|
-
export { RecipesResource, } from "./resources/recipes.js";
|
|
17
|
+
export { type RecipeGraphReference, type RecipeGraphValidationResult, type RecipeRunOptions, type RecipeRunOutput, type RecipeRunResult, RecipesResource, } from "./resources/recipes.js";
|
|
18
18
|
export { ScenariosResource, } from "./resources/scenarios.js";
|
|
19
19
|
export { SqlResource, } from "./resources/sql.js";
|
|
20
20
|
export { VariablesResource, } from "./resources/variables.js";
|
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
import type { ConnectionSummary } from "../schemas.js";
|
|
2
2
|
import { BaseResource } from "./base.js";
|
|
3
|
+
export interface ConnectionSchemaListOptions {
|
|
4
|
+
connection: string;
|
|
5
|
+
projectKey?: string;
|
|
6
|
+
}
|
|
7
|
+
export interface ConnectionTableListOptions extends ConnectionSchemaListOptions {
|
|
8
|
+
catalog?: string;
|
|
9
|
+
schema?: string;
|
|
10
|
+
}
|
|
3
11
|
export declare class ConnectionsResource extends BaseResource {
|
|
4
12
|
/**
|
|
5
13
|
* Returns sorted list of all connection names visible to the current user.
|
|
@@ -19,4 +27,6 @@ export declare class ConnectionsResource extends BaseResource {
|
|
|
19
27
|
mode?: "fast" | "rich";
|
|
20
28
|
projectKey?: string;
|
|
21
29
|
}): Promise<ConnectionSummary[]>;
|
|
30
|
+
schemas(opts: ConnectionSchemaListOptions): Promise<string[]>;
|
|
31
|
+
tables(opts: ConnectionTableListOptions): Promise<Record<string, unknown>>;
|
|
22
32
|
}
|
|
@@ -79,4 +79,20 @@ export class ConnectionsResource extends BaseResource {
|
|
|
79
79
|
}
|
|
80
80
|
return inferRichConnectionsFromDatasets(this.client, projectEnc);
|
|
81
81
|
}
|
|
82
|
+
async schemas(opts) {
|
|
83
|
+
const pk = this.resolveProjectKey(opts.projectKey);
|
|
84
|
+
const params = new URLSearchParams();
|
|
85
|
+
params.set("connectionName", opts.connection);
|
|
86
|
+
return this.client.get(`/public/api/projects/${encodeURIComponent(pk)}/datasets/tables-import/actions/list-schemas?${params.toString()}`);
|
|
87
|
+
}
|
|
88
|
+
async tables(opts) {
|
|
89
|
+
const pk = this.resolveProjectKey(opts.projectKey);
|
|
90
|
+
const params = new URLSearchParams();
|
|
91
|
+
params.set("connectionName", opts.connection);
|
|
92
|
+
if (opts.catalog !== undefined)
|
|
93
|
+
params.set("catalogName", opts.catalog);
|
|
94
|
+
if (opts.schema !== undefined)
|
|
95
|
+
params.set("schemaName", opts.schema);
|
|
96
|
+
return this.client.get(`/public/api/projects/${encodeURIComponent(pk)}/datasets/tables-import/actions/list-tables?${params.toString()}`);
|
|
97
|
+
}
|
|
82
98
|
}
|
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
import { BaseResource } from "./base.js";
|
|
2
2
|
import type { DatasetCreateOptions, DatasetDetails, DatasetSchema, DatasetSummary } from "../schemas.js";
|
|
3
|
+
export interface DatasetBuildValidationResult {
|
|
4
|
+
valid: boolean;
|
|
5
|
+
datasetName: string;
|
|
6
|
+
projectKey: string;
|
|
7
|
+
type: string | null;
|
|
8
|
+
path: string | null;
|
|
9
|
+
formatType: string | null;
|
|
10
|
+
warnings: string[];
|
|
11
|
+
}
|
|
12
|
+
export interface DatasetSchemaColumnInput {
|
|
13
|
+
name: string;
|
|
14
|
+
type: string;
|
|
15
|
+
comment?: string;
|
|
16
|
+
}
|
|
3
17
|
/**
|
|
4
18
|
* Compare streamed TSV header columns against a known dataset schema.
|
|
5
19
|
* Returns an array of warning strings (empty if all columns match).
|
|
@@ -14,6 +28,8 @@ export declare class DatasetsResource extends BaseResource {
|
|
|
14
28
|
get(datasetName: string, projectKey?: string): Promise<DatasetDetails>;
|
|
15
29
|
/** Get dataset schema (column names and types). */
|
|
16
30
|
schema(datasetName: string, projectKey?: string): Promise<DatasetSchema>;
|
|
31
|
+
/** Replace dataset schema columns directly through the schema endpoint. */
|
|
32
|
+
updateSchema(datasetName: string, columns: DatasetSchemaColumnInput[], projectKey?: string): Promise<void>;
|
|
17
33
|
/**
|
|
18
34
|
* Preview dataset data as CSV text.
|
|
19
35
|
* Streams TSV from the API, converts to CSV, and returns up to `maxRows`
|
|
@@ -51,6 +67,8 @@ export declare class DatasetsResource extends BaseResource {
|
|
|
51
67
|
* from existing datasets on the same connection.
|
|
52
68
|
*/
|
|
53
69
|
create(opts: DatasetCreateOptions): Promise<Record<string, unknown>>;
|
|
70
|
+
/** Validate common build blockers before running a dataset build. */
|
|
71
|
+
validateBuildSettings(datasetName: string, projectKey?: string): Promise<DatasetBuildValidationResult>;
|
|
54
72
|
/** Update a dataset by deep-merging a patch into the current definition. */
|
|
55
73
|
update(datasetName: string, data: Record<string, unknown>, projectKey?: string): Promise<void>;
|
|
56
74
|
/** Delete a dataset. */
|
|
@@ -327,6 +327,11 @@ export class DatasetsResource extends BaseResource {
|
|
|
327
327
|
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/datasets/${dsEnc}/schema`);
|
|
328
328
|
return this.client.safeParse(DatasetSchemaSchema, raw, "datasets.schema");
|
|
329
329
|
}
|
|
330
|
+
/** Replace dataset schema columns directly through the schema endpoint. */
|
|
331
|
+
async updateSchema(datasetName, columns, projectKey) {
|
|
332
|
+
const dsEnc = encodeURIComponent(datasetName);
|
|
333
|
+
await this.client.put(`/public/api/projects/${this.enc(projectKey)}/datasets/${dsEnc}/schema`, { columns, });
|
|
334
|
+
}
|
|
330
335
|
/**
|
|
331
336
|
* Preview dataset data as CSV text.
|
|
332
337
|
* Streams TSV from the API, converts to CSV, and returns up to `maxRows`
|
|
@@ -444,6 +449,42 @@ export class DatasetsResource extends BaseResource {
|
|
|
444
449
|
return this.client.post(`/public/api/projects/${enc}/datasets/`, body);
|
|
445
450
|
}
|
|
446
451
|
}
|
|
452
|
+
/** Validate common build blockers before running a dataset build. */
|
|
453
|
+
async validateBuildSettings(datasetName, projectKey) {
|
|
454
|
+
const pk = this.resolveProjectKey(projectKey);
|
|
455
|
+
const details = await this.get(datasetName, pk);
|
|
456
|
+
const params = details.params ?? {};
|
|
457
|
+
const type = details.type ?? null;
|
|
458
|
+
const path = typeof params.path === "string" && params.path.trim().length > 0
|
|
459
|
+
? params.path
|
|
460
|
+
: null;
|
|
461
|
+
const table = typeof params.table === "string" && params.table.trim().length > 0
|
|
462
|
+
? params.table
|
|
463
|
+
: null;
|
|
464
|
+
const normalizedType = (type ?? "").toLowerCase();
|
|
465
|
+
const fileBacked = !table
|
|
466
|
+
&& (normalizedType.includes("filesystem")
|
|
467
|
+
|| normalizedType.includes("uploaded")
|
|
468
|
+
|| normalizedType.includes("s3")
|
|
469
|
+
|| path !== null);
|
|
470
|
+
const formatType = details.formatType ?? null;
|
|
471
|
+
const warnings = [];
|
|
472
|
+
if (fileBacked && !path) {
|
|
473
|
+
warnings.push("File-backed dataset has no writable storage path configured.");
|
|
474
|
+
}
|
|
475
|
+
if (fileBacked && !formatType) {
|
|
476
|
+
warnings.push("File-backed dataset has no formatType configured.");
|
|
477
|
+
}
|
|
478
|
+
return {
|
|
479
|
+
valid: warnings.length === 0,
|
|
480
|
+
datasetName,
|
|
481
|
+
projectKey: pk,
|
|
482
|
+
type,
|
|
483
|
+
path,
|
|
484
|
+
formatType,
|
|
485
|
+
warnings,
|
|
486
|
+
};
|
|
487
|
+
}
|
|
447
488
|
/** Update a dataset by deep-merging a patch into the current definition. */
|
|
448
489
|
async update(datasetName, data, projectKey) {
|
|
449
490
|
const dsEnc = encodeURIComponent(datasetName);
|
|
@@ -1,6 +1,34 @@
|
|
|
1
1
|
import type { BuildMode, JobSummary, JobWaitResult } from "../schemas.js";
|
|
2
2
|
import { BaseResource } from "./base.js";
|
|
3
3
|
export type JobBuildTargetType = "DATASET" | "MANAGED_FOLDER";
|
|
4
|
+
export type JobLogFilter = "stdout" | "stderr" | "user" | "errors";
|
|
5
|
+
export interface JobLogSummary {
|
|
6
|
+
state: string;
|
|
7
|
+
lineCount: number;
|
|
8
|
+
lines: string[];
|
|
9
|
+
}
|
|
10
|
+
export interface JobBuildTarget {
|
|
11
|
+
id: string;
|
|
12
|
+
type?: JobBuildTargetType;
|
|
13
|
+
projectKey?: string;
|
|
14
|
+
partition?: string;
|
|
15
|
+
}
|
|
16
|
+
export interface JobBuildOptions {
|
|
17
|
+
buildMode?: BuildMode;
|
|
18
|
+
autoUpdateSchema?: boolean;
|
|
19
|
+
projectKey?: string;
|
|
20
|
+
targetType?: JobBuildTargetType;
|
|
21
|
+
partition?: string;
|
|
22
|
+
}
|
|
23
|
+
export interface JobBuildAndWaitOptions extends JobBuildOptions {
|
|
24
|
+
activity?: string;
|
|
25
|
+
includeLogs?: boolean;
|
|
26
|
+
maxLogLines?: number;
|
|
27
|
+
pollIntervalMs?: number;
|
|
28
|
+
timeoutMs?: number;
|
|
29
|
+
logFilter?: JobLogFilter;
|
|
30
|
+
summary?: boolean;
|
|
31
|
+
}
|
|
4
32
|
interface ComputeNextPollDelayMsOptions {
|
|
5
33
|
pollCount: number;
|
|
6
34
|
baseIntervalMs: number;
|
|
@@ -28,32 +56,33 @@ export declare class JobsResource extends BaseResource {
|
|
|
28
56
|
projectKey?: string;
|
|
29
57
|
}): Promise<string>;
|
|
30
58
|
/**
|
|
31
|
-
* Start a build job for
|
|
59
|
+
* Start a build job for one or more dataset or managed-folder outputs.
|
|
32
60
|
* Returns the new job's ID.
|
|
33
61
|
*/
|
|
34
|
-
|
|
35
|
-
buildMode?: BuildMode;
|
|
36
|
-
autoUpdateSchema?: boolean;
|
|
37
|
-
projectKey?: string;
|
|
38
|
-
targetType?: JobBuildTargetType;
|
|
39
|
-
}): Promise<{
|
|
62
|
+
buildOutputs(targets: JobBuildTarget[], opts?: JobBuildOptions): Promise<{
|
|
40
63
|
jobId: string;
|
|
41
64
|
}>;
|
|
65
|
+
/**
|
|
66
|
+
* Start a build job for a single dataset or managed folder.
|
|
67
|
+
* Returns the new job's ID.
|
|
68
|
+
*/
|
|
69
|
+
build(targetId: string, opts?: JobBuildOptions): Promise<{
|
|
70
|
+
jobId: string;
|
|
71
|
+
}>;
|
|
72
|
+
/**
|
|
73
|
+
* Build one or more dataset or managed-folder outputs and wait for a terminal state.
|
|
74
|
+
* Combines {@link buildOutputs} then {@link wait}.
|
|
75
|
+
*/
|
|
76
|
+
buildAndWaitOutputs(targets: JobBuildTarget[], opts?: JobBuildAndWaitOptions): Promise<JobWaitResult & {
|
|
77
|
+
logSummary?: JobLogSummary;
|
|
78
|
+
}>;
|
|
42
79
|
/**
|
|
43
80
|
* Build a dataset or managed folder and wait for the job to reach a terminal state.
|
|
44
81
|
* Combines {@link build} then {@link wait}.
|
|
45
82
|
*/
|
|
46
|
-
buildAndWait(targetId: string, opts?: {
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
activity?: string;
|
|
50
|
-
includeLogs?: boolean;
|
|
51
|
-
maxLogLines?: number;
|
|
52
|
-
pollIntervalMs?: number;
|
|
53
|
-
timeoutMs?: number;
|
|
54
|
-
projectKey?: string;
|
|
55
|
-
targetType?: JobBuildTargetType;
|
|
56
|
-
}): Promise<JobWaitResult>;
|
|
83
|
+
buildAndWait(targetId: string, opts?: JobBuildAndWaitOptions): Promise<JobWaitResult & {
|
|
84
|
+
logSummary?: JobLogSummary;
|
|
85
|
+
}>;
|
|
57
86
|
/**
|
|
58
87
|
* Poll a job until it reaches a terminal state or times out.
|
|
59
88
|
*
|
|
@@ -65,11 +94,15 @@ export declare class JobsResource extends BaseResource {
|
|
|
65
94
|
wait(jobId: string, opts?: {
|
|
66
95
|
activity?: string;
|
|
67
96
|
includeLogs?: boolean;
|
|
97
|
+
logFilter?: JobLogFilter;
|
|
68
98
|
maxLogLines?: number;
|
|
69
99
|
pollIntervalMs?: number;
|
|
100
|
+
summary?: boolean;
|
|
70
101
|
timeoutMs?: number;
|
|
71
102
|
projectKey?: string;
|
|
72
|
-
}): Promise<JobWaitResult
|
|
103
|
+
}): Promise<JobWaitResult & {
|
|
104
|
+
logSummary?: JobLogSummary;
|
|
105
|
+
}>;
|
|
73
106
|
/** Request a job abort. */
|
|
74
107
|
abort(jobId: string, projectKey?: string): Promise<void>;
|
|
75
108
|
}
|
|
@@ -35,6 +35,84 @@ export function computeNextPollDelayMs({ pollCount, baseIntervalMs, adaptiveEnab
|
|
|
35
35
|
function sleep(ms) {
|
|
36
36
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
37
37
|
}
|
|
38
|
+
const DEFAULT_TARGET_PARTITION = "NP";
|
|
39
|
+
function jobBuildOutput(target, defaultProjectKey, defaultPartition, defaultTargetType) {
|
|
40
|
+
const targetType = target.type ?? defaultTargetType ?? "DATASET";
|
|
41
|
+
const projectKey = target.projectKey ?? defaultProjectKey;
|
|
42
|
+
const partition = target.partition ?? defaultPartition;
|
|
43
|
+
const output = { projectKey, id: target.id, type: targetType, };
|
|
44
|
+
if (targetType === "DATASET") {
|
|
45
|
+
if (partition !== undefined)
|
|
46
|
+
output.partition = partition;
|
|
47
|
+
}
|
|
48
|
+
else {
|
|
49
|
+
output.targetManagedFolderProjectKey = projectKey;
|
|
50
|
+
output.targetManagedFolder = target.id;
|
|
51
|
+
output.targetPartition = partition ?? DEFAULT_TARGET_PARTITION;
|
|
52
|
+
}
|
|
53
|
+
return output;
|
|
54
|
+
}
|
|
55
|
+
function jobBuildDefinition(targets, defaultProjectKey, opts) {
|
|
56
|
+
if (targets.length === 0) {
|
|
57
|
+
throw new Error("At least one build target is required.");
|
|
58
|
+
}
|
|
59
|
+
const payload = {
|
|
60
|
+
outputs: targets.map((target) => jobBuildOutput(target, defaultProjectKey, opts?.partition, opts?.targetType)),
|
|
61
|
+
type: opts?.buildMode ?? "NON_RECURSIVE_FORCED_BUILD",
|
|
62
|
+
};
|
|
63
|
+
if (opts?.autoUpdateSchema
|
|
64
|
+
&& targets.every((target) => (target.type ?? opts?.targetType ?? "DATASET") === "DATASET")) {
|
|
65
|
+
payload.autoUpdateSchemaBeforeEachRecipeRun = true;
|
|
66
|
+
}
|
|
67
|
+
return payload;
|
|
68
|
+
}
|
|
69
|
+
function jobLogLines(log) {
|
|
70
|
+
return log.split(/\r?\n/).map((line) => line.trimEnd());
|
|
71
|
+
}
|
|
72
|
+
function lineMatchesLogFilter(line, filter) {
|
|
73
|
+
const normalized = line.toLowerCase();
|
|
74
|
+
switch (filter) {
|
|
75
|
+
case "stdout":
|
|
76
|
+
return normalized.includes("stdout") || line.startsWith(">>> ");
|
|
77
|
+
case "stderr":
|
|
78
|
+
return normalized.includes("stderr");
|
|
79
|
+
case "errors":
|
|
80
|
+
return /\b(error|failed|failure|exception|traceback)\b/i.test(line);
|
|
81
|
+
case "user":
|
|
82
|
+
return !/^\d{4}[-/]\d{2}[-/]\d{2}/.test(line)
|
|
83
|
+
&& !normalized.includes("backend-log")
|
|
84
|
+
&& !normalized.includes("debug");
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
function filterJobLog(log, filter) {
|
|
88
|
+
if (!filter)
|
|
89
|
+
return log;
|
|
90
|
+
return jobLogLines(log).filter((line) => lineMatchesLogFilter(line, filter)).join("\n");
|
|
91
|
+
}
|
|
92
|
+
function limitJobLog(log, maxLines) {
|
|
93
|
+
if (!log)
|
|
94
|
+
return "";
|
|
95
|
+
const limit = maxLines ?? DEFAULT_MAX_LOG_LINES;
|
|
96
|
+
if (limit === 0 || limit === -1)
|
|
97
|
+
return log;
|
|
98
|
+
const lines = log.split(/\r?\n/);
|
|
99
|
+
const hasTrailingLineBreak = lines.length > 0 && lines[lines.length - 1] === "";
|
|
100
|
+
if (hasTrailingLineBreak)
|
|
101
|
+
lines.pop();
|
|
102
|
+
if (lines.length <= limit)
|
|
103
|
+
return log;
|
|
104
|
+
const tail = lines.slice(-Math.max(1, limit)).join("\n");
|
|
105
|
+
return hasTrailingLineBreak ? `${tail}\n` : tail;
|
|
106
|
+
}
|
|
107
|
+
function summarizeJobLog(state, log, maxLines) {
|
|
108
|
+
const lines = jobLogLines(log).map((line) => line.trim()).filter((line) => line.length > 0);
|
|
109
|
+
const summaryLines = lines.slice(-Math.max(1, maxLines));
|
|
110
|
+
return {
|
|
111
|
+
state,
|
|
112
|
+
lineCount: lines.length,
|
|
113
|
+
lines: summaryLines,
|
|
114
|
+
};
|
|
115
|
+
}
|
|
38
116
|
export class JobsResource extends BaseResource {
|
|
39
117
|
/** List jobs in a project. */
|
|
40
118
|
async list(projectKey) {
|
|
@@ -56,56 +134,58 @@ export class JobsResource extends BaseResource {
|
|
|
56
134
|
const jobEnc = encodeURIComponent(jobId);
|
|
57
135
|
const query = opts?.activity ? `?activity=${encodeURIComponent(opts.activity)}` : "";
|
|
58
136
|
const log = await this.client.getText(`/public/api/projects/${this.enc(opts?.projectKey)}/jobs/${jobEnc}/log/${query}`);
|
|
59
|
-
|
|
60
|
-
return "";
|
|
61
|
-
const limit = opts?.maxLogLines ?? DEFAULT_MAX_LOG_LINES;
|
|
62
|
-
if (limit === 0 || limit === -1) {
|
|
63
|
-
return log;
|
|
64
|
-
}
|
|
65
|
-
const lines = log.split("\n");
|
|
66
|
-
if (lines.length > limit) {
|
|
67
|
-
return lines.slice(-limit).join("\n");
|
|
68
|
-
}
|
|
69
|
-
return log;
|
|
137
|
+
return limitJobLog(log, opts?.maxLogLines);
|
|
70
138
|
}
|
|
71
139
|
/**
|
|
72
|
-
* Start a build job for
|
|
140
|
+
* Start a build job for one or more dataset or managed-folder outputs.
|
|
73
141
|
* Returns the new job's ID.
|
|
74
142
|
*/
|
|
75
|
-
async
|
|
143
|
+
async buildOutputs(targets, opts) {
|
|
76
144
|
const pk = this.resolveProjectKey(opts?.projectKey);
|
|
77
145
|
const enc = encodeURIComponent(pk);
|
|
78
|
-
const
|
|
79
|
-
const jobDef = {
|
|
80
|
-
outputs: [{ projectKey: pk, id: targetId, type: targetType, },],
|
|
81
|
-
type: opts?.buildMode ?? "NON_RECURSIVE_FORCED_BUILD",
|
|
82
|
-
};
|
|
83
|
-
if (opts?.autoUpdateSchema && targetType === "DATASET") {
|
|
84
|
-
jobDef.autoUpdateSchemaBeforeEachRecipeRun = true;
|
|
85
|
-
}
|
|
146
|
+
const jobDef = jobBuildDefinition(targets, pk, opts);
|
|
86
147
|
const job = await this.client.post(`/public/api/projects/${enc}/jobs/`, jobDef);
|
|
87
148
|
return { jobId: job.id, };
|
|
88
149
|
}
|
|
89
150
|
/**
|
|
90
|
-
*
|
|
91
|
-
*
|
|
151
|
+
* Start a build job for a single dataset or managed folder.
|
|
152
|
+
* Returns the new job's ID.
|
|
92
153
|
*/
|
|
93
|
-
async
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
154
|
+
async build(targetId, opts) {
|
|
155
|
+
return this.buildOutputs([{
|
|
156
|
+
id: targetId,
|
|
157
|
+
type: opts?.targetType,
|
|
158
|
+
partition: opts?.partition,
|
|
159
|
+
},], opts);
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* Build one or more dataset or managed-folder outputs and wait for a terminal state.
|
|
163
|
+
* Combines {@link buildOutputs} then {@link wait}.
|
|
164
|
+
*/
|
|
165
|
+
async buildAndWaitOutputs(targets, opts) {
|
|
166
|
+
const { jobId, } = await this.buildOutputs(targets, opts);
|
|
100
167
|
return this.wait(jobId, {
|
|
101
168
|
activity: opts?.activity,
|
|
102
169
|
includeLogs: opts?.includeLogs,
|
|
170
|
+
logFilter: opts?.logFilter,
|
|
103
171
|
maxLogLines: opts?.maxLogLines,
|
|
104
172
|
pollIntervalMs: opts?.pollIntervalMs,
|
|
173
|
+
summary: opts?.summary,
|
|
105
174
|
timeoutMs: opts?.timeoutMs,
|
|
106
175
|
projectKey: opts?.projectKey,
|
|
107
176
|
});
|
|
108
177
|
}
|
|
178
|
+
/**
|
|
179
|
+
* Build a dataset or managed folder and wait for the job to reach a terminal state.
|
|
180
|
+
* Combines {@link build} then {@link wait}.
|
|
181
|
+
*/
|
|
182
|
+
async buildAndWait(targetId, opts) {
|
|
183
|
+
return this.buildAndWaitOutputs([{
|
|
184
|
+
id: targetId,
|
|
185
|
+
type: opts?.targetType,
|
|
186
|
+
partition: opts?.partition,
|
|
187
|
+
},], opts);
|
|
188
|
+
}
|
|
109
189
|
/**
|
|
110
190
|
* Poll a job until it reaches a terminal state or times out.
|
|
111
191
|
*
|
|
@@ -133,12 +213,19 @@ export class JobsResource extends BaseResource {
|
|
|
133
213
|
if (isTerminalState(state)) {
|
|
134
214
|
const success = isSuccessfulTerminalState(state);
|
|
135
215
|
let log;
|
|
136
|
-
|
|
137
|
-
|
|
216
|
+
let logSummary;
|
|
217
|
+
if (opts?.includeLogs || opts?.summary) {
|
|
218
|
+
const rawLog = await this.log(jobId, {
|
|
138
219
|
activity: opts.activity,
|
|
139
|
-
maxLogLines: opts.maxLogLines,
|
|
220
|
+
maxLogLines: opts.summary ? 0 : opts.maxLogLines,
|
|
140
221
|
projectKey: opts.projectKey,
|
|
141
222
|
});
|
|
223
|
+
const filteredLog = filterJobLog(rawLog, opts.logFilter);
|
|
224
|
+
if (opts.includeLogs)
|
|
225
|
+
log = limitJobLog(filteredLog, opts.maxLogLines);
|
|
226
|
+
if (opts.summary) {
|
|
227
|
+
logSummary = summarizeJobLog(state, filteredLog, opts.maxLogLines ?? 20);
|
|
228
|
+
}
|
|
142
229
|
}
|
|
143
230
|
return {
|
|
144
231
|
success,
|
|
@@ -154,6 +241,7 @@ export class JobsResource extends BaseResource {
|
|
|
154
241
|
total: gs.total ?? null,
|
|
155
242
|
},
|
|
156
243
|
...(log !== undefined ? { log, } : {}),
|
|
244
|
+
...(logSummary !== undefined ? { logSummary, } : {}),
|
|
157
245
|
};
|
|
158
246
|
}
|
|
159
247
|
// Timeout — return failure result, don't throw
|
|
@@ -10,8 +10,9 @@ export declare class NotebooksResource extends BaseResource {
|
|
|
10
10
|
/**
|
|
11
11
|
* Delete a Jupyter notebook.
|
|
12
12
|
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
13
|
+
* The SDK intentionally does not expose notebook creation as a first-class
|
|
14
|
+
* resource method yet; integration tests create disposable notebooks through
|
|
15
|
+
* the documented project-level create endpoint.
|
|
15
16
|
*/
|
|
16
17
|
deleteJupyter(name: string, projectKey?: string): Promise<void>;
|
|
17
18
|
/**
|
|
@@ -26,8 +27,9 @@ export declare class NotebooksResource extends BaseResource {
|
|
|
26
27
|
/**
|
|
27
28
|
* Unload (stop) a running Jupyter notebook session.
|
|
28
29
|
*
|
|
29
|
-
* DSS public APIs
|
|
30
|
-
*
|
|
30
|
+
* DSS public APIs expose session listing and unloading, but this SDK has no
|
|
31
|
+
* path to start a disposable kernel session; live unload coverage therefore
|
|
32
|
+
* requires an externally-started disposable session fixture.
|
|
31
33
|
*/
|
|
32
34
|
unloadJupyter(name: string, sessionId: string, projectKey?: string): Promise<void>;
|
|
33
35
|
/** List all SQL notebooks in a project. */
|
|
@@ -21,8 +21,9 @@ export class NotebooksResource extends BaseResource {
|
|
|
21
21
|
/**
|
|
22
22
|
* Delete a Jupyter notebook.
|
|
23
23
|
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
24
|
+
* The SDK intentionally does not expose notebook creation as a first-class
|
|
25
|
+
* resource method yet; integration tests create disposable notebooks through
|
|
26
|
+
* the documented project-level create endpoint.
|
|
26
27
|
*/
|
|
27
28
|
async deleteJupyter(name, projectKey) {
|
|
28
29
|
const nameEnc = encodeURIComponent(name);
|
|
@@ -52,8 +53,9 @@ export class NotebooksResource extends BaseResource {
|
|
|
52
53
|
/**
|
|
53
54
|
* Unload (stop) a running Jupyter notebook session.
|
|
54
55
|
*
|
|
55
|
-
* DSS public APIs
|
|
56
|
-
*
|
|
56
|
+
* DSS public APIs expose session listing and unloading, but this SDK has no
|
|
57
|
+
* path to start a disposable kernel session; live unload coverage therefore
|
|
58
|
+
* requires an externally-started disposable session fixture.
|
|
57
59
|
*/
|
|
58
60
|
async unloadJupyter(name, sessionId, projectKey) {
|
|
59
61
|
const nameEnc = encodeURIComponent(name);
|
|
@@ -1,5 +1,47 @@
|
|
|
1
|
-
import type { RecipeCreateOptions, RecipeCreateResult, RecipeSummary } from "../schemas.js";
|
|
1
|
+
import type { BuildMode, JobWaitResult, RecipeCreateOptions, RecipeCreateResult, RecipeSummary } from "../schemas.js";
|
|
2
2
|
import { BaseResource } from "./base.js";
|
|
3
|
+
import type { JobBuildTarget, JobBuildTargetType, JobLogFilter, JobLogSummary } from "./jobs.js";
|
|
4
|
+
export interface RecipeRunOutput extends JobBuildTarget {
|
|
5
|
+
ref: string;
|
|
6
|
+
role: string;
|
|
7
|
+
}
|
|
8
|
+
export interface RecipeGraphReference {
|
|
9
|
+
ref: string;
|
|
10
|
+
role: string;
|
|
11
|
+
type?: JobBuildTargetType;
|
|
12
|
+
exists: boolean;
|
|
13
|
+
id?: string;
|
|
14
|
+
}
|
|
15
|
+
export interface RecipeGraphValidationResult {
|
|
16
|
+
valid: boolean;
|
|
17
|
+
recipeName: string;
|
|
18
|
+
projectKey: string;
|
|
19
|
+
inputs: RecipeGraphReference[];
|
|
20
|
+
outputs: RecipeGraphReference[];
|
|
21
|
+
missingInputs: RecipeGraphReference[];
|
|
22
|
+
missingOutputs: RecipeGraphReference[];
|
|
23
|
+
ambiguousOutputs: string[];
|
|
24
|
+
warnings: string[];
|
|
25
|
+
}
|
|
26
|
+
export interface RecipeRunOptions {
|
|
27
|
+
buildMode?: BuildMode;
|
|
28
|
+
includeLogs?: boolean;
|
|
29
|
+
maxLogLines?: number;
|
|
30
|
+
partition?: string;
|
|
31
|
+
pollIntervalMs?: number;
|
|
32
|
+
projectKey?: string;
|
|
33
|
+
wait?: boolean;
|
|
34
|
+
timeoutMs?: number;
|
|
35
|
+
logFilter?: JobLogFilter;
|
|
36
|
+
summary?: boolean;
|
|
37
|
+
}
|
|
38
|
+
export type RecipeRunResult = {
|
|
39
|
+
logSummary?: JobLogSummary;
|
|
40
|
+
recipeName: string;
|
|
41
|
+
outputs: RecipeRunOutput[];
|
|
42
|
+
} & ({
|
|
43
|
+
jobId: string;
|
|
44
|
+
} | JobWaitResult);
|
|
3
45
|
export declare class RecipesResource extends BaseResource {
|
|
4
46
|
/** List all recipes in a project. */
|
|
5
47
|
list(projectKey?: string): Promise<RecipeSummary[]>;
|
|
@@ -15,6 +57,17 @@ export declare class RecipesResource extends BaseResource {
|
|
|
15
57
|
recipe: Record<string, unknown>;
|
|
16
58
|
payload?: string;
|
|
17
59
|
}>;
|
|
60
|
+
/** Validate declared recipe graph references before running/building. */
|
|
61
|
+
validateGraph(recipeName: string, opts?: {
|
|
62
|
+
projectKey?: string;
|
|
63
|
+
}): Promise<RecipeGraphValidationResult>;
|
|
64
|
+
/** Resolve recipe outputs to job-build targets. */
|
|
65
|
+
resolveRunOutputs(recipeName: string, opts?: {
|
|
66
|
+
partition?: string;
|
|
67
|
+
projectKey?: string;
|
|
68
|
+
}): Promise<RecipeRunOutput[]>;
|
|
69
|
+
/** Run a recipe by building its resolved outputs. */
|
|
70
|
+
run(recipeName: string, opts?: RecipeRunOptions): Promise<RecipeRunResult>;
|
|
18
71
|
/** Create a recipe, with optional output dataset provisioning and join configuration. */
|
|
19
72
|
create(opts: RecipeCreateOptions): Promise<RecipeCreateResult>;
|
|
20
73
|
/**
|