dataiku-sdk 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/dss.js +2 -0
- package/dist/packages/types/src/index.d.ts +458 -0
- package/dist/packages/types/src/index.js +384 -0
- package/dist/src/cli.d.ts +2 -0
- package/dist/src/cli.js +689 -0
- package/dist/src/client.d.ts +89 -0
- package/dist/src/client.js +301 -0
- package/dist/src/errors.d.ts +29 -0
- package/dist/src/errors.js +141 -0
- package/dist/src/index.d.ts +20 -0
- package/dist/src/index.js +24 -0
- package/dist/src/resources/base.d.ts +7 -0
- package/dist/src/resources/base.js +12 -0
- package/dist/src/resources/code-envs.d.ts +8 -0
- package/dist/src/resources/code-envs.js +36 -0
- package/dist/src/resources/connections.d.ts +20 -0
- package/dist/src/resources/connections.js +77 -0
- package/dist/src/resources/datasets.d.ts +57 -0
- package/dist/src/resources/datasets.js +423 -0
- package/dist/src/resources/folders.d.ts +15 -0
- package/dist/src/resources/folders.js +58 -0
- package/dist/src/resources/jobs.d.ts +72 -0
- package/dist/src/resources/jobs.js +184 -0
- package/dist/src/resources/notebooks.d.ts +34 -0
- package/dist/src/resources/notebooks.js +75 -0
- package/dist/src/resources/projects.d.ts +38 -0
- package/dist/src/resources/projects.js +185 -0
- package/dist/src/resources/recipes.d.ts +35 -0
- package/dist/src/resources/recipes.js +281 -0
- package/dist/src/resources/scenarios.d.ts +26 -0
- package/dist/src/resources/scenarios.js +57 -0
- package/dist/src/resources/sql.d.ts +40 -0
- package/dist/src/resources/sql.js +40 -0
- package/dist/src/resources/variables.d.ts +10 -0
- package/dist/src/resources/variables.js +22 -0
- package/dist/src/schemas.d.ts +7 -0
- package/dist/src/schemas.js +6 -0
- package/dist/src/utils/deep-merge.d.ts +1 -0
- package/dist/src/utils/deep-merge.js +15 -0
- package/dist/src/utils/flow-map.d.ts +37 -0
- package/dist/src/utils/flow-map.js +296 -0
- package/dist/src/utils/pagination.d.ts +8 -0
- package/dist/src/utils/pagination.js +23 -0
- package/dist/src/utils/sanitize.d.ts +2 -0
- package/dist/src/utils/sanitize.js +16 -0
- package/package.json +47 -0
- package/packages/types/dist/index.d.ts +458 -0
- package/packages/types/dist/index.js +384 -0
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import type { BuildMode, JobSummary, JobWaitResult } from "../schemas.js";
|
|
2
|
+
import { BaseResource } from "./base.js";
|
|
3
|
+
interface ComputeNextPollDelayMsOptions {
|
|
4
|
+
pollCount: number;
|
|
5
|
+
baseIntervalMs: number;
|
|
6
|
+
adaptiveEnabled: boolean;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Compute the next poll delay.
|
|
10
|
+
* When adaptive polling is enabled, the interval doubles every 3 polls,
|
|
11
|
+
* capped at MAX_POLL_INTERVAL_MS (or baseIntervalMs if it's larger).
|
|
12
|
+
*/
|
|
13
|
+
export declare function computeNextPollDelayMs({ pollCount, baseIntervalMs, adaptiveEnabled, }: ComputeNextPollDelayMsOptions): number;
|
|
14
|
+
export declare class JobsResource extends BaseResource {
|
|
15
|
+
/** List jobs in a project. */
|
|
16
|
+
list(projectKey?: string): Promise<JobSummary[]>;
|
|
17
|
+
/** Get full details for a single job. */
|
|
18
|
+
get(jobId: string, projectKey?: string): Promise<Record<string, unknown>>;
|
|
19
|
+
/**
|
|
20
|
+
* Retrieve job log text.
|
|
21
|
+
* Returns the last `maxLogLines` lines (default 50) from the tail.
|
|
22
|
+
*/
|
|
23
|
+
log(jobId: string, opts?: {
|
|
24
|
+
activity?: string;
|
|
25
|
+
maxLogLines?: number;
|
|
26
|
+
projectKey?: string;
|
|
27
|
+
}): Promise<string>;
|
|
28
|
+
/**
|
|
29
|
+
* Start a dataset build job.
|
|
30
|
+
* Returns the new job's ID.
|
|
31
|
+
*/
|
|
32
|
+
build(datasetName: string, opts?: {
|
|
33
|
+
buildMode?: BuildMode;
|
|
34
|
+
autoUpdateSchema?: boolean;
|
|
35
|
+
projectKey?: string;
|
|
36
|
+
}): Promise<{
|
|
37
|
+
jobId: string;
|
|
38
|
+
}>;
|
|
39
|
+
/**
|
|
40
|
+
* Build a dataset and wait for the job to reach a terminal state.
|
|
41
|
+
* Combines {@link build} then {@link wait}.
|
|
42
|
+
*/
|
|
43
|
+
buildAndWait(datasetName: string, opts?: {
|
|
44
|
+
buildMode?: BuildMode;
|
|
45
|
+
autoUpdateSchema?: boolean;
|
|
46
|
+
activity?: string;
|
|
47
|
+
includeLogs?: boolean;
|
|
48
|
+
maxLogLines?: number;
|
|
49
|
+
pollIntervalMs?: number;
|
|
50
|
+
timeoutMs?: number;
|
|
51
|
+
projectKey?: string;
|
|
52
|
+
}): Promise<JobWaitResult>;
|
|
53
|
+
/**
|
|
54
|
+
* Poll a job until it reaches a terminal state or times out.
|
|
55
|
+
*
|
|
56
|
+
* Adaptive polling doubles the interval every 3 polls when
|
|
57
|
+
* `pollIntervalMs` is not explicitly set.
|
|
58
|
+
*
|
|
59
|
+
* On timeout, returns `{ success: false, ... }` rather than throwing.
|
|
60
|
+
*/
|
|
61
|
+
wait(jobId: string, opts?: {
|
|
62
|
+
activity?: string;
|
|
63
|
+
includeLogs?: boolean;
|
|
64
|
+
maxLogLines?: number;
|
|
65
|
+
pollIntervalMs?: number;
|
|
66
|
+
timeoutMs?: number;
|
|
67
|
+
projectKey?: string;
|
|
68
|
+
}): Promise<JobWaitResult>;
|
|
69
|
+
/** Request a job abort. */
|
|
70
|
+
abort(jobId: string, projectKey?: string): Promise<void>;
|
|
71
|
+
}
|
|
72
|
+
export {};
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
import { JobSummaryArraySchema, } from "../schemas.js";
|
|
2
|
+
import { BaseResource, } from "./base.js";
|
|
3
|
+
const DEFAULT_POLL_INTERVAL_MS = 2_000;
|
|
4
|
+
const MAX_POLL_INTERVAL_MS = 10_000;
|
|
5
|
+
const DEFAULT_TIMEOUT_MS = 120_000;
|
|
6
|
+
const DEFAULT_MAX_LOG_LINES = 50;
|
|
7
|
+
const TERMINAL_STATES = new Set([
|
|
8
|
+
"DONE",
|
|
9
|
+
"FAILED",
|
|
10
|
+
"ABORTED",
|
|
11
|
+
"KILLED",
|
|
12
|
+
"CANCELED",
|
|
13
|
+
"CANCELLED",
|
|
14
|
+
"ERROR",
|
|
15
|
+
]);
|
|
16
|
+
function isTerminalState(state) {
|
|
17
|
+
return TERMINAL_STATES.has((state ?? "").toUpperCase());
|
|
18
|
+
}
|
|
19
|
+
function isSuccessfulTerminalState(state) {
|
|
20
|
+
return (state ?? "").toUpperCase() === "DONE";
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Compute the next poll delay.
|
|
24
|
+
* When adaptive polling is enabled, the interval doubles every 3 polls,
|
|
25
|
+
* capped at MAX_POLL_INTERVAL_MS (or baseIntervalMs if it's larger).
|
|
26
|
+
*/
|
|
27
|
+
export function computeNextPollDelayMs({ pollCount, baseIntervalMs, adaptiveEnabled, }) {
|
|
28
|
+
if (!adaptiveEnabled) {
|
|
29
|
+
return baseIntervalMs;
|
|
30
|
+
}
|
|
31
|
+
const step = Math.max(0, Math.floor((pollCount - 1) / 3));
|
|
32
|
+
const interval = baseIntervalMs * 2 ** step;
|
|
33
|
+
return Math.min(interval, Math.max(baseIntervalMs, MAX_POLL_INTERVAL_MS));
|
|
34
|
+
}
|
|
35
|
+
function sleep(ms) {
|
|
36
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
37
|
+
}
|
|
38
|
+
export class JobsResource extends BaseResource {
|
|
39
|
+
/** List jobs in a project. */
|
|
40
|
+
async list(projectKey) {
|
|
41
|
+
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/jobs/`);
|
|
42
|
+
return this.client.safeParse(JobSummaryArraySchema, raw, "jobs.list");
|
|
43
|
+
}
|
|
44
|
+
/** Get full details for a single job. */
|
|
45
|
+
async get(jobId, projectKey) {
|
|
46
|
+
const jobEnc = encodeURIComponent(jobId);
|
|
47
|
+
// Trailing slash required — DSS Cloud proxy misroutes URLs ending in .NNN (job ID timestamps)
|
|
48
|
+
return this.client.get(`/public/api/projects/${this.enc(projectKey)}/jobs/${jobEnc}/`);
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Retrieve job log text.
|
|
52
|
+
* Returns the last `maxLogLines` lines (default 50) from the tail.
|
|
53
|
+
*/
|
|
54
|
+
async log(jobId, opts) {
|
|
55
|
+
const jobEnc = encodeURIComponent(jobId);
|
|
56
|
+
const query = opts?.activity ? `?activity=${encodeURIComponent(opts.activity)}` : "";
|
|
57
|
+
const log = await this.client.getText(`/public/api/projects/${this.enc(opts?.projectKey)}/jobs/${jobEnc}/log/${query}`);
|
|
58
|
+
if (!log)
|
|
59
|
+
return "";
|
|
60
|
+
const lines = log.split("\n");
|
|
61
|
+
const limit = opts?.maxLogLines ?? DEFAULT_MAX_LOG_LINES;
|
|
62
|
+
if (lines.length > limit) {
|
|
63
|
+
return lines.slice(-limit).join("\n");
|
|
64
|
+
}
|
|
65
|
+
return log;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Start a dataset build job.
|
|
69
|
+
* Returns the new job's ID.
|
|
70
|
+
*/
|
|
71
|
+
async build(datasetName, opts) {
|
|
72
|
+
const pk = this.resolveProjectKey(opts?.projectKey);
|
|
73
|
+
const enc = encodeURIComponent(pk);
|
|
74
|
+
const jobDef = {
|
|
75
|
+
outputs: [{ projectKey: pk, id: datasetName, type: "DATASET", },],
|
|
76
|
+
type: opts?.buildMode ?? "NON_RECURSIVE_FORCED_BUILD",
|
|
77
|
+
};
|
|
78
|
+
if (opts?.autoUpdateSchema) {
|
|
79
|
+
jobDef.autoUpdateSchemaBeforeEachRecipeRun = true;
|
|
80
|
+
}
|
|
81
|
+
const job = await this.client.post(`/public/api/projects/${enc}/jobs/`, jobDef);
|
|
82
|
+
return { jobId: job.id, };
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Build a dataset and wait for the job to reach a terminal state.
|
|
86
|
+
* Combines {@link build} then {@link wait}.
|
|
87
|
+
*/
|
|
88
|
+
async buildAndWait(datasetName, opts) {
|
|
89
|
+
const { jobId, } = await this.build(datasetName, {
|
|
90
|
+
buildMode: opts?.buildMode,
|
|
91
|
+
autoUpdateSchema: opts?.autoUpdateSchema,
|
|
92
|
+
projectKey: opts?.projectKey,
|
|
93
|
+
});
|
|
94
|
+
return this.wait(jobId, {
|
|
95
|
+
activity: opts?.activity,
|
|
96
|
+
includeLogs: opts?.includeLogs,
|
|
97
|
+
maxLogLines: opts?.maxLogLines,
|
|
98
|
+
pollIntervalMs: opts?.pollIntervalMs,
|
|
99
|
+
timeoutMs: opts?.timeoutMs,
|
|
100
|
+
projectKey: opts?.projectKey,
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Poll a job until it reaches a terminal state or times out.
|
|
105
|
+
*
|
|
106
|
+
* Adaptive polling doubles the interval every 3 polls when
|
|
107
|
+
* `pollIntervalMs` is not explicitly set.
|
|
108
|
+
*
|
|
109
|
+
* On timeout, returns `{ success: false, ... }` rather than throwing.
|
|
110
|
+
*/
|
|
111
|
+
async wait(jobId, opts) {
|
|
112
|
+
const projectEnc = this.enc(opts?.projectKey);
|
|
113
|
+
const jobEnc = encodeURIComponent(jobId);
|
|
114
|
+
const baseIntervalMs = Math.max(1, opts?.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS);
|
|
115
|
+
const adaptivePolling = opts?.pollIntervalMs === undefined;
|
|
116
|
+
const timeout = Math.max(baseIntervalMs, opts?.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
|
117
|
+
const startedAt = Date.now();
|
|
118
|
+
let pollCount = 0;
|
|
119
|
+
while (true) {
|
|
120
|
+
pollCount += 1;
|
|
121
|
+
const j = await this.client.get(`/public/api/projects/${projectEnc}/jobs/${jobEnc}/`);
|
|
122
|
+
const bs = j.baseStatus ?? {};
|
|
123
|
+
const def = bs.def ?? {};
|
|
124
|
+
const gs = j.globalState ?? {};
|
|
125
|
+
const state = bs.state ?? "unknown";
|
|
126
|
+
const elapsedMs = Date.now() - startedAt;
|
|
127
|
+
if (isTerminalState(state)) {
|
|
128
|
+
const success = isSuccessfulTerminalState(state);
|
|
129
|
+
let log;
|
|
130
|
+
if (opts?.includeLogs) {
|
|
131
|
+
log = await this.log(jobId, {
|
|
132
|
+
activity: opts.activity,
|
|
133
|
+
maxLogLines: opts.maxLogLines,
|
|
134
|
+
projectKey: opts.projectKey,
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
return {
|
|
138
|
+
success,
|
|
139
|
+
jobId: def.id ?? jobId,
|
|
140
|
+
state,
|
|
141
|
+
type: def.type ?? "unknown",
|
|
142
|
+
elapsedMs,
|
|
143
|
+
pollCount,
|
|
144
|
+
progress: {
|
|
145
|
+
done: gs.done ?? 0,
|
|
146
|
+
failed: gs.failed ?? 0,
|
|
147
|
+
running: gs.running ?? 0,
|
|
148
|
+
total: gs.total ?? null,
|
|
149
|
+
},
|
|
150
|
+
...(log !== undefined ? { log, } : {}),
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
// Timeout — return failure result, don't throw
|
|
154
|
+
if (elapsedMs >= timeout) {
|
|
155
|
+
return {
|
|
156
|
+
success: false,
|
|
157
|
+
jobId,
|
|
158
|
+
state,
|
|
159
|
+
type: def.type ?? "unknown",
|
|
160
|
+
elapsedMs,
|
|
161
|
+
pollCount,
|
|
162
|
+
timedOut: true,
|
|
163
|
+
progress: {
|
|
164
|
+
done: gs.done ?? 0,
|
|
165
|
+
failed: gs.failed ?? 0,
|
|
166
|
+
running: gs.running ?? 0,
|
|
167
|
+
total: gs.total ?? null,
|
|
168
|
+
},
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
const nextDelayMs = computeNextPollDelayMs({
|
|
172
|
+
pollCount,
|
|
173
|
+
baseIntervalMs,
|
|
174
|
+
adaptiveEnabled: adaptivePolling,
|
|
175
|
+
});
|
|
176
|
+
await sleep(Math.min(nextDelayMs, timeout - elapsedMs));
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
/** Request a job abort. */
|
|
180
|
+
async abort(jobId, projectKey) {
|
|
181
|
+
const jobEnc = encodeURIComponent(jobId);
|
|
182
|
+
await this.client.post(`/public/api/projects/${this.enc(projectKey)}/jobs/${jobEnc}/abort/`);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { JupyterNotebookContent, JupyterNotebookSummary, NotebookSession, SqlNotebookContent, SqlNotebookSummary } from "../schemas.js";
|
|
2
|
+
import { BaseResource } from "./base.js";
|
|
3
|
+
export declare class NotebooksResource extends BaseResource {
|
|
4
|
+
/** List all Jupyter notebooks in a project. */
|
|
5
|
+
listJupyter(projectKey?: string): Promise<JupyterNotebookSummary[]>;
|
|
6
|
+
/** Get the full content of a Jupyter notebook. */
|
|
7
|
+
getJupyter(name: string, projectKey?: string): Promise<JupyterNotebookContent>;
|
|
8
|
+
/** Save (overwrite) a Jupyter notebook's content. */
|
|
9
|
+
saveJupyter(name: string, content: JupyterNotebookContent, projectKey?: string): Promise<void>;
|
|
10
|
+
/** Delete a Jupyter notebook. */
|
|
11
|
+
deleteJupyter(name: string, projectKey?: string): Promise<void>;
|
|
12
|
+
/** Clear all cell outputs from a Jupyter notebook. */
|
|
13
|
+
clearJupyterOutputs(name: string, projectKey?: string): Promise<void>;
|
|
14
|
+
/** List running kernel sessions for a Jupyter notebook. */
|
|
15
|
+
listJupyterSessions(name: string, projectKey?: string): Promise<NotebookSession[]>;
|
|
16
|
+
/** Unload (stop) a running Jupyter notebook session. */
|
|
17
|
+
unloadJupyter(name: string, sessionId: string, projectKey?: string): Promise<void>;
|
|
18
|
+
/** List all SQL notebooks in a project. */
|
|
19
|
+
listSql(projectKey?: string): Promise<SqlNotebookSummary[]>;
|
|
20
|
+
/** Get the full content of a SQL notebook. */
|
|
21
|
+
getSql(id: string, projectKey?: string): Promise<SqlNotebookContent>;
|
|
22
|
+
/** Save (overwrite) a SQL notebook's content. */
|
|
23
|
+
saveSql(id: string, content: SqlNotebookContent, projectKey?: string): Promise<void>;
|
|
24
|
+
/** Delete a SQL notebook. */
|
|
25
|
+
deleteSql(id: string, projectKey?: string): Promise<void>;
|
|
26
|
+
/** Get execution history for a SQL notebook (keyed by cell ID). */
|
|
27
|
+
getSqlHistory(id: string, projectKey?: string): Promise<Record<string, unknown[]>>;
|
|
28
|
+
/** Clear execution history for a SQL notebook. */
|
|
29
|
+
clearSqlHistory(id: string, opts?: {
|
|
30
|
+
cellId?: string;
|
|
31
|
+
numRunsToRetain?: number;
|
|
32
|
+
projectKey?: string;
|
|
33
|
+
}): Promise<void>;
|
|
34
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { JupyterNotebookContentSchema, JupyterNotebookSummaryArraySchema, NotebookSessionArraySchema, SqlNotebookContentSchema, SqlNotebookSummaryArraySchema, } from "../schemas.js";
|
|
2
|
+
import { BaseResource, } from "./base.js";
|
|
3
|
+
export class NotebooksResource extends BaseResource {
|
|
4
|
+
// ── Jupyter Notebooks ──────────────────────────────────────────────
|
|
5
|
+
/** List all Jupyter notebooks in a project. */
|
|
6
|
+
async listJupyter(projectKey) {
|
|
7
|
+
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/jupyter-notebooks/`);
|
|
8
|
+
return this.client.safeParse(JupyterNotebookSummaryArraySchema, raw, "notebooks.listJupyter");
|
|
9
|
+
}
|
|
10
|
+
/** Get the full content of a Jupyter notebook. */
|
|
11
|
+
async getJupyter(name, projectKey) {
|
|
12
|
+
const nameEnc = encodeURIComponent(name);
|
|
13
|
+
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/jupyter-notebooks/${nameEnc}`);
|
|
14
|
+
return this.client.safeParse(JupyterNotebookContentSchema, raw, "notebooks.getJupyter");
|
|
15
|
+
}
|
|
16
|
+
/** Save (overwrite) a Jupyter notebook's content. */
|
|
17
|
+
async saveJupyter(name, content, projectKey) {
|
|
18
|
+
const nameEnc = encodeURIComponent(name);
|
|
19
|
+
await this.client.putVoid(`/public/api/projects/${this.enc(projectKey)}/jupyter-notebooks/${nameEnc}`, content);
|
|
20
|
+
}
|
|
21
|
+
/** Delete a Jupyter notebook. */
|
|
22
|
+
async deleteJupyter(name, projectKey) {
|
|
23
|
+
const nameEnc = encodeURIComponent(name);
|
|
24
|
+
await this.client.del(`/public/api/projects/${this.enc(projectKey)}/jupyter-notebooks/${nameEnc}`);
|
|
25
|
+
}
|
|
26
|
+
/** Clear all cell outputs from a Jupyter notebook. */
|
|
27
|
+
async clearJupyterOutputs(name, projectKey) {
|
|
28
|
+
const nameEnc = encodeURIComponent(name);
|
|
29
|
+
await this.client.del(`/public/api/projects/${this.enc(projectKey)}/jupyter-notebooks/${nameEnc}/outputs`);
|
|
30
|
+
}
|
|
31
|
+
/** List running kernel sessions for a Jupyter notebook. */
|
|
32
|
+
async listJupyterSessions(name, projectKey) {
|
|
33
|
+
const nameEnc = encodeURIComponent(name);
|
|
34
|
+
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/jupyter-notebooks/${nameEnc}/sessions`);
|
|
35
|
+
return this.client.safeParse(NotebookSessionArraySchema, raw, "notebooks.sessionsJupyter");
|
|
36
|
+
}
|
|
37
|
+
/** Unload (stop) a running Jupyter notebook session. */
|
|
38
|
+
async unloadJupyter(name, sessionId, projectKey) {
|
|
39
|
+
const nameEnc = encodeURIComponent(name);
|
|
40
|
+
const sidEnc = encodeURIComponent(sessionId);
|
|
41
|
+
await this.client.del(`/public/api/projects/${this.enc(projectKey)}/jupyter-notebooks/${nameEnc}/sessions/${sidEnc}`);
|
|
42
|
+
}
|
|
43
|
+
// ── SQL Notebooks ──────────────────────────────────────────────────
|
|
44
|
+
/** List all SQL notebooks in a project. */
|
|
45
|
+
async listSql(projectKey) {
|
|
46
|
+
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/sql-notebooks/`);
|
|
47
|
+
return this.client.safeParse(SqlNotebookSummaryArraySchema, raw, "notebooks.listSql");
|
|
48
|
+
}
|
|
49
|
+
/** Get the full content of a SQL notebook. */
|
|
50
|
+
async getSql(id, projectKey) {
|
|
51
|
+
const idEnc = encodeURIComponent(id);
|
|
52
|
+
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/sql-notebooks/${idEnc}`);
|
|
53
|
+
return this.client.safeParse(SqlNotebookContentSchema, raw, "notebooks.getSql");
|
|
54
|
+
}
|
|
55
|
+
/** Save (overwrite) a SQL notebook's content. */
|
|
56
|
+
async saveSql(id, content, projectKey) {
|
|
57
|
+
const idEnc = encodeURIComponent(id);
|
|
58
|
+
await this.client.putVoid(`/public/api/projects/${this.enc(projectKey)}/sql-notebooks/${idEnc}`, content);
|
|
59
|
+
}
|
|
60
|
+
/** Delete a SQL notebook. */
|
|
61
|
+
async deleteSql(id, projectKey) {
|
|
62
|
+
const idEnc = encodeURIComponent(id);
|
|
63
|
+
await this.client.del(`/public/api/projects/${this.enc(projectKey)}/sql-notebooks/${idEnc}`);
|
|
64
|
+
}
|
|
65
|
+
/** Get execution history for a SQL notebook (keyed by cell ID). */
|
|
66
|
+
async getSqlHistory(id, projectKey) {
|
|
67
|
+
const idEnc = encodeURIComponent(id);
|
|
68
|
+
return this.client.get(`/public/api/projects/${this.enc(projectKey)}/sql-notebooks/${idEnc}/history`);
|
|
69
|
+
}
|
|
70
|
+
/** Clear execution history for a SQL notebook. */
|
|
71
|
+
async clearSqlHistory(id, opts) {
|
|
72
|
+
const idEnc = encodeURIComponent(id);
|
|
73
|
+
await this.client.post(`/public/api/projects/${this.enc(opts?.projectKey)}/sql-notebooks/${idEnc}/history/clear`, { cellId: opts?.cellId, numRunsToRetain: opts?.numRunsToRetain, });
|
|
74
|
+
}
|
|
75
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import type { FlowMapOptions, ProjectDetails, ProjectMetadata, ProjectSummary } from "../schemas.js";
|
|
2
|
+
import type { NormalizedFlowMap } from "../utils/flow-map.js";
|
|
3
|
+
import { BaseResource } from "./base.js";
|
|
4
|
+
interface FlowMapTruncationSummary {
|
|
5
|
+
truncated: boolean;
|
|
6
|
+
maxNodes: number | null;
|
|
7
|
+
maxEdges: number | null;
|
|
8
|
+
nodeCountBefore: number;
|
|
9
|
+
nodeCountAfter: number;
|
|
10
|
+
edgeCountBefore: number;
|
|
11
|
+
edgeCountAfter: number;
|
|
12
|
+
}
|
|
13
|
+
export interface FlowMapResult {
|
|
14
|
+
map: NormalizedFlowMap;
|
|
15
|
+
truncation: FlowMapTruncationSummary;
|
|
16
|
+
raw?: unknown;
|
|
17
|
+
}
|
|
18
|
+
export declare class ProjectsResource extends BaseResource {
|
|
19
|
+
/** List all projects visible to the API key. */
|
|
20
|
+
list(): Promise<ProjectSummary[]>;
|
|
21
|
+
/** Get details for a single project. */
|
|
22
|
+
get(projectKey?: string): Promise<ProjectDetails>;
|
|
23
|
+
/** Get metadata (tags, custom fields, checklists) for a project. */
|
|
24
|
+
metadata(projectKey?: string): Promise<ProjectMetadata>;
|
|
25
|
+
/** Get the raw flow graph for a project. */
|
|
26
|
+
flow(projectKey?: string): Promise<unknown>;
|
|
27
|
+
/**
|
|
28
|
+
* Build a normalized, optionally truncated flow map for a project.
|
|
29
|
+
*
|
|
30
|
+
* Fetches the flow graph and supplementary metadata (datasets, recipes,
|
|
31
|
+
* managed folders) in parallel. Folder name resolution uses a timeout
|
|
32
|
+
* to avoid blocking when the folders endpoint is slow.
|
|
33
|
+
*/
|
|
34
|
+
map(opts?: FlowMapOptions & {
|
|
35
|
+
projectKey?: string;
|
|
36
|
+
}): Promise<FlowMapResult>;
|
|
37
|
+
}
|
|
38
|
+
export {};
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
import { ProjectDetailsSchema, ProjectMetadataSchema, ProjectSummaryArraySchema, } from "../schemas.js";
|
|
2
|
+
import { normalizeFlowGraph, } from "../utils/flow-map.js";
|
|
3
|
+
import { BaseResource, } from "./base.js";
|
|
4
|
+
function fetchWithTimeout(label, timeoutMs, fetcher) {
|
|
5
|
+
return new Promise((resolve) => {
|
|
6
|
+
let settled = false;
|
|
7
|
+
const timer = setTimeout(() => {
|
|
8
|
+
if (settled)
|
|
9
|
+
return;
|
|
10
|
+
settled = true;
|
|
11
|
+
resolve({
|
|
12
|
+
warning: `${label} metadata timed out after ${timeoutMs}ms; continuing without it.`,
|
|
13
|
+
});
|
|
14
|
+
}, timeoutMs);
|
|
15
|
+
fetcher().then((value) => {
|
|
16
|
+
if (settled)
|
|
17
|
+
return;
|
|
18
|
+
settled = true;
|
|
19
|
+
clearTimeout(timer);
|
|
20
|
+
resolve({ value, });
|
|
21
|
+
}, (error) => {
|
|
22
|
+
if (settled)
|
|
23
|
+
return;
|
|
24
|
+
settled = true;
|
|
25
|
+
clearTimeout(timer);
|
|
26
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
27
|
+
resolve({
|
|
28
|
+
warning: `${label} metadata unavailable: ${detail}`,
|
|
29
|
+
});
|
|
30
|
+
});
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
function computeRootsAndLeaves(nodes, edges) {
|
|
34
|
+
const inDegree = new Map();
|
|
35
|
+
const outDegree = new Map();
|
|
36
|
+
for (const node of nodes) {
|
|
37
|
+
inDegree.set(node.id, 0);
|
|
38
|
+
outDegree.set(node.id, 0);
|
|
39
|
+
}
|
|
40
|
+
for (const edge of edges) {
|
|
41
|
+
inDegree.set(edge.to, (inDegree.get(edge.to) ?? 0) + 1);
|
|
42
|
+
outDegree.set(edge.from, (outDegree.get(edge.from) ?? 0) + 1);
|
|
43
|
+
}
|
|
44
|
+
const roots = nodes
|
|
45
|
+
.filter((node) => (inDegree.get(node.id) ?? 0) === 0)
|
|
46
|
+
.map((node) => node.id)
|
|
47
|
+
.sort((a, b) => a.localeCompare(b));
|
|
48
|
+
const leaves = nodes
|
|
49
|
+
.filter((node) => (outDegree.get(node.id) ?? 0) === 0)
|
|
50
|
+
.map((node) => node.id)
|
|
51
|
+
.sort((a, b) => a.localeCompare(b));
|
|
52
|
+
return { roots, leaves, };
|
|
53
|
+
}
|
|
54
|
+
function truncateFlowMap(normalized, maxNodes, maxEdges) {
|
|
55
|
+
const nodes = maxNodes === undefined ? normalized.nodes : normalized.nodes.slice(0, maxNodes);
|
|
56
|
+
const nodeIds = new Set(nodes.map((node) => node.id));
|
|
57
|
+
const edgesWithinNodes = normalized.edges.filter((edge) => nodeIds.has(edge.from) && nodeIds.has(edge.to));
|
|
58
|
+
const edges = maxEdges === undefined ? edgesWithinNodes : edgesWithinNodes.slice(0, maxEdges);
|
|
59
|
+
const { roots, leaves, } = computeRootsAndLeaves(nodes, edges);
|
|
60
|
+
const truncation = {
|
|
61
|
+
truncated: nodes.length < normalized.nodes.length || edges.length < normalized.edges.length,
|
|
62
|
+
maxNodes: maxNodes ?? null,
|
|
63
|
+
maxEdges: maxEdges ?? null,
|
|
64
|
+
nodeCountBefore: normalized.nodes.length,
|
|
65
|
+
nodeCountAfter: nodes.length,
|
|
66
|
+
edgeCountBefore: normalized.edges.length,
|
|
67
|
+
edgeCountAfter: edges.length,
|
|
68
|
+
};
|
|
69
|
+
const warnings = truncation.truncated
|
|
70
|
+
? [
|
|
71
|
+
...normalized.warnings,
|
|
72
|
+
`Flow map truncated (nodes ${truncation.nodeCountAfter}/${truncation.nodeCountBefore}, edges ${truncation.edgeCountAfter}/${truncation.edgeCountBefore}).`,
|
|
73
|
+
]
|
|
74
|
+
: normalized.warnings;
|
|
75
|
+
return {
|
|
76
|
+
map: {
|
|
77
|
+
...normalized,
|
|
78
|
+
nodes,
|
|
79
|
+
edges,
|
|
80
|
+
roots,
|
|
81
|
+
leaves,
|
|
82
|
+
warnings,
|
|
83
|
+
stats: {
|
|
84
|
+
nodeCount: nodes.length,
|
|
85
|
+
edgeCount: edges.length,
|
|
86
|
+
datasets: nodes.filter((node) => node.kind === "dataset").length,
|
|
87
|
+
recipes: nodes.filter((node) => node.kind === "recipe").length,
|
|
88
|
+
roots: roots.length,
|
|
89
|
+
leaves: leaves.length,
|
|
90
|
+
},
|
|
91
|
+
},
|
|
92
|
+
truncation,
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
// ---------------------------------------------------------------------------
|
|
96
|
+
// Resource
|
|
97
|
+
// ---------------------------------------------------------------------------
|
|
98
|
+
const DEFAULT_MAX_NODES = 300;
|
|
99
|
+
const DEFAULT_MAX_EDGES = 600;
|
|
100
|
+
const DEFAULT_METADATA_TIMEOUT_MS = 1_500;
|
|
101
|
+
export class ProjectsResource extends BaseResource {
|
|
102
|
+
/** List all projects visible to the API key. */
|
|
103
|
+
async list() {
|
|
104
|
+
const raw = await this.client.get("/public/api/projects/");
|
|
105
|
+
return this.client.safeParse(ProjectSummaryArraySchema, raw, "projects.list");
|
|
106
|
+
}
|
|
107
|
+
/** Get details for a single project. */
|
|
108
|
+
async get(projectKey) {
|
|
109
|
+
const enc = this.enc(projectKey);
|
|
110
|
+
const raw = await this.client.get(`/public/api/projects/${enc}/`);
|
|
111
|
+
return this.client.safeParse(ProjectDetailsSchema, raw, "projects.get");
|
|
112
|
+
}
|
|
113
|
+
/** Get metadata (tags, custom fields, checklists) for a project. */
|
|
114
|
+
async metadata(projectKey) {
|
|
115
|
+
const enc = this.enc(projectKey);
|
|
116
|
+
const raw = await this.client.get(`/public/api/projects/${enc}/metadata`);
|
|
117
|
+
return this.client.safeParse(ProjectMetadataSchema, raw, "projects.metadata");
|
|
118
|
+
}
|
|
119
|
+
/** Get the raw flow graph for a project. */
|
|
120
|
+
async flow(projectKey) {
|
|
121
|
+
const enc = this.enc(projectKey);
|
|
122
|
+
return this.client.get(`/public/api/projects/${enc}/flow/graph/`);
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Build a normalized, optionally truncated flow map for a project.
|
|
126
|
+
*
|
|
127
|
+
* Fetches the flow graph and supplementary metadata (datasets, recipes,
|
|
128
|
+
* managed folders) in parallel. Folder name resolution uses a timeout
|
|
129
|
+
* to avoid blocking when the folders endpoint is slow.
|
|
130
|
+
*/
|
|
131
|
+
async map(opts) {
|
|
132
|
+
const enc = this.enc(opts?.projectKey);
|
|
133
|
+
const pk = this.resolveProjectKey(opts?.projectKey);
|
|
134
|
+
const timeoutMs = DEFAULT_METADATA_TIMEOUT_MS;
|
|
135
|
+
const [rawGraph, foldersMeta, datasetsMeta, recipesMeta,] = await Promise.all([
|
|
136
|
+
this.client.get(`/public/api/projects/${enc}/flow/graph/`),
|
|
137
|
+
fetchWithTimeout("Managed folders", timeoutMs, () => this.client.get(`/public/api/projects/${enc}/managedfolders/`)),
|
|
138
|
+
fetchWithTimeout("Datasets", timeoutMs, () => this.client.get(`/public/api/projects/${enc}/datasets/`)),
|
|
139
|
+
fetchWithTimeout("Recipes", timeoutMs, () => this.client.get(`/public/api/projects/${enc}/recipes/`)),
|
|
140
|
+
]);
|
|
141
|
+
// Build folder name lookup
|
|
142
|
+
const folderNamesById = {};
|
|
143
|
+
const allFolderIds = [];
|
|
144
|
+
for (const f of foldersMeta.value ?? []) {
|
|
145
|
+
if (!f.id || f.id.length === 0)
|
|
146
|
+
continue;
|
|
147
|
+
allFolderIds.push(f.id);
|
|
148
|
+
folderNamesById[f.id] = f.name ?? f.id;
|
|
149
|
+
}
|
|
150
|
+
const allDatasetNames = (datasetsMeta.value ?? [])
|
|
151
|
+
.map((d) => d.name)
|
|
152
|
+
.filter((n) => typeof n === "string" && n.length > 0);
|
|
153
|
+
const allRecipeNames = (recipesMeta.value ?? [])
|
|
154
|
+
.map((r) => r.name)
|
|
155
|
+
.filter((n) => typeof n === "string" && n.length > 0);
|
|
156
|
+
// Normalize the flow graph
|
|
157
|
+
const normalizedBase = normalizeFlowGraph(rawGraph, pk, {
|
|
158
|
+
folderNamesById,
|
|
159
|
+
allDatasetNames,
|
|
160
|
+
allRecipeNames,
|
|
161
|
+
allFolderIds,
|
|
162
|
+
});
|
|
163
|
+
// Append any metadata fetch warnings
|
|
164
|
+
const metadataWarnings = [
|
|
165
|
+
foldersMeta.warning,
|
|
166
|
+
datasetsMeta.warning,
|
|
167
|
+
recipesMeta.warning,
|
|
168
|
+
].filter((w) => typeof w === "string" && w.length > 0);
|
|
169
|
+
const normalized = metadataWarnings.length > 0
|
|
170
|
+
? {
|
|
171
|
+
...normalizedBase,
|
|
172
|
+
warnings: [...normalizedBase.warnings, ...metadataWarnings,],
|
|
173
|
+
}
|
|
174
|
+
: normalizedBase;
|
|
175
|
+
// Truncate
|
|
176
|
+
const effectiveMaxNodes = opts?.maxNodes ?? DEFAULT_MAX_NODES;
|
|
177
|
+
const effectiveMaxEdges = opts?.maxEdges ?? DEFAULT_MAX_EDGES;
|
|
178
|
+
const { map, truncation, } = truncateFlowMap(normalized, effectiveMaxNodes, effectiveMaxEdges);
|
|
179
|
+
const result = { map, truncation, };
|
|
180
|
+
if (opts?.includeRaw) {
|
|
181
|
+
result.raw = rawGraph;
|
|
182
|
+
}
|
|
183
|
+
return result;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { RecipeCreateOptions, RecipeCreateResult, RecipeSummary } from "../schemas.js";
|
|
2
|
+
import { BaseResource } from "./base.js";
|
|
3
|
+
export declare class RecipesResource extends BaseResource {
|
|
4
|
+
/** List all recipes in a project. */
|
|
5
|
+
list(projectKey?: string): Promise<RecipeSummary[]>;
|
|
6
|
+
/**
|
|
7
|
+
* Get a recipe definition (and optionally its payload).
|
|
8
|
+
* Returns the raw API response shape: `{ recipe, payload }`.
|
|
9
|
+
*/
|
|
10
|
+
get(recipeName: string, opts?: {
|
|
11
|
+
includePayload?: boolean;
|
|
12
|
+
payloadMaxLines?: number;
|
|
13
|
+
projectKey?: string;
|
|
14
|
+
}): Promise<{
|
|
15
|
+
recipe: Record<string, unknown>;
|
|
16
|
+
payload?: string;
|
|
17
|
+
}>;
|
|
18
|
+
/** Create a recipe, with optional output dataset provisioning and join configuration. */
|
|
19
|
+
create(opts: RecipeCreateOptions): Promise<RecipeCreateResult>;
|
|
20
|
+
/**
|
|
21
|
+
* Update a recipe by merging the patch into the current definition.
|
|
22
|
+
* The `recipe` sub-object is spread-merged to preserve nested fields.
|
|
23
|
+
*/
|
|
24
|
+
update(recipeName: string, data: Record<string, unknown>, projectKey?: string): Promise<void>;
|
|
25
|
+
/** Delete a recipe. */
|
|
26
|
+
delete(recipeName: string, projectKey?: string): Promise<void>;
|
|
27
|
+
/**
|
|
28
|
+
* Download a recipe definition as a JSON file.
|
|
29
|
+
* Returns the path to the written file.
|
|
30
|
+
*/
|
|
31
|
+
download(recipeName: string, opts?: {
|
|
32
|
+
outputPath?: string;
|
|
33
|
+
projectKey?: string;
|
|
34
|
+
}): Promise<string>;
|
|
35
|
+
}
|