dataiku-sdk 3.0.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +246 -19
- package/dist/build-metadata.json +1 -1
- package/dist/packages/types/src/index.d.ts +39 -4
- package/dist/packages/types/src/index.js +31 -2
- package/dist/src/auth.js +1 -1
- package/dist/src/cli/coerce.js +11 -2
- package/dist/src/cli/commands/analysis.js +4 -3
- package/dist/src/cli/commands/api-service.js +66 -18
- package/dist/src/cli/commands/app.js +4 -3
- package/dist/src/cli/commands/bundle.js +51 -16
- package/dist/src/cli/commands/code-env.js +219 -57
- package/dist/src/cli/commands/code.d.ts +19 -0
- package/dist/src/cli/commands/code.js +87 -15
- package/dist/src/cli/commands/dashboard.js +7 -6
- package/dist/src/cli/commands/data-quality.js +9 -8
- package/dist/src/cli/commands/dataset.js +17 -11
- package/dist/src/cli/commands/doctor.js +4 -2
- package/dist/src/cli/commands/flow-zone.js +42 -13
- package/dist/src/cli/commands/folder.js +9 -8
- package/dist/src/cli/commands/future.js +2 -1
- package/dist/src/cli/commands/insight.js +8 -7
- package/dist/src/cli/commands/job.js +4 -3
- package/dist/src/cli/commands/meaning.js +5 -4
- package/dist/src/cli/commands/ml-task.js +3 -2
- package/dist/src/cli/commands/model-evaluation-store.js +4 -3
- package/dist/src/cli/commands/notebook.js +82 -50
- package/dist/src/cli/commands/project-git.js +8 -5
- package/dist/src/cli/commands/project-library.d.ts +21 -0
- package/dist/src/cli/commands/project-library.js +128 -56
- package/dist/src/cli/commands/project.js +19 -4
- package/dist/src/cli/commands/recipe.js +51 -25
- package/dist/src/cli/commands/saved-model.js +4 -3
- package/dist/src/cli/commands/scenario.js +9 -8
- package/dist/src/cli/commands/sql.d.ts +10 -0
- package/dist/src/cli/commands/sql.js +42 -19
- package/dist/src/cli/commands/statistics.js +7 -6
- package/dist/src/cli/commands/variable.js +2 -1
- package/dist/src/cli/commands/webapp.js +68 -13
- package/dist/src/cli/commands/wiki.js +7 -6
- package/dist/src/cli/contract.d.ts +16 -1
- package/dist/src/cli/contract.js +967 -186
- package/dist/src/cli/env.d.ts +9 -0
- package/dist/src/cli/env.js +20 -2
- package/dist/src/cli/flags.d.ts +4 -0
- package/dist/src/cli/flags.js +29 -1
- package/dist/src/cli/helpers/cleanup.js +104 -3
- package/dist/src/cli/helpers/flow-zone.d.ts +3 -0
- package/dist/src/cli/helpers/flow-zone.js +33 -3
- package/dist/src/cli/helpers/recipe.d.ts +20 -0
- package/dist/src/cli/helpers/recipe.js +158 -3
- package/dist/src/cli/output.d.ts +2 -0
- package/dist/src/cli/output.js +33 -6
- package/dist/src/cli/runtime.d.ts +35 -1
- package/dist/src/cli/runtime.js +125 -18
- package/dist/src/cli/types.d.ts +7 -0
- package/dist/src/cli.js +226 -95
- package/dist/src/client.d.ts +63 -1
- package/dist/src/client.js +305 -53
- package/dist/src/errors.d.ts +8 -2
- package/dist/src/errors.js +42 -0
- package/dist/src/index.d.ts +4 -3
- package/dist/src/index.js +2 -1
- package/dist/src/resources/api-services.d.ts +48 -6
- package/dist/src/resources/api-services.js +54 -8
- package/dist/src/resources/bundles.d.ts +51 -6
- package/dist/src/resources/bundles.js +39 -9
- package/dist/src/resources/code-envs.d.ts +25 -3
- package/dist/src/resources/code-envs.js +104 -2
- package/dist/src/resources/datasets.d.ts +20 -0
- package/dist/src/resources/datasets.js +232 -52
- package/dist/src/resources/folders.js +2 -1
- package/dist/src/resources/jobs.d.ts +35 -10
- package/dist/src/resources/jobs.js +92 -17
- package/dist/src/resources/notebooks.d.ts +77 -7
- package/dist/src/resources/notebooks.js +143 -19
- package/dist/src/resources/project-git.d.ts +12 -0
- package/dist/src/resources/project-git.js +39 -8
- package/dist/src/resources/project-library.d.ts +122 -7
- package/dist/src/resources/project-library.js +494 -25
- package/dist/src/resources/projects.d.ts +5 -4
- package/dist/src/resources/projects.js +21 -19
- package/dist/src/resources/scenarios.d.ts +30 -0
- package/dist/src/resources/scenarios.js +73 -27
- package/dist/src/resources/sql.d.ts +6 -0
- package/dist/src/resources/sql.js +23 -3
- package/dist/src/resources/webapps.d.ts +35 -4
- package/dist/src/resources/webapps.js +54 -5
- package/dist/src/schemas.d.ts +2 -2
- package/dist/src/schemas.js +1 -1
- package/dist/src/skill.d.ts +18 -0
- package/dist/src/skill.js +93 -9
- package/dist/src/utils/flow-analysis.d.ts +53 -0
- package/dist/src/utils/flow-analysis.js +382 -0
- package/dist/src/utils/project-archive.d.ts +41 -2
- package/dist/src/utils/project-archive.js +159 -16
- package/dist/src/utils/response-file.d.ts +2 -0
- package/dist/src/utils/response-file.js +34 -0
- package/package.json +9 -7
- package/packages/types/dist/index.d.ts +39 -4
- package/packages/types/dist/index.js +31 -2
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +44 -197
- package/skills/dataiku-dss/references/app-releases.md +27 -0
- package/skills/dataiku-dss/references/authentication.md +39 -0
- package/skills/dataiku-dss/references/coding.md +8 -0
- package/skills/dataiku-dss/references/discovery.md +30 -0
- package/skills/dataiku-dss/references/flow-maps.md +12 -0
- package/skills/dataiku-dss/references/mutations.md +33 -0
- package/skills/dataiku-dss/references/troubleshooting.md +29 -0
package/dist/src/skill.js
CHANGED
|
@@ -1,17 +1,22 @@
|
|
|
1
|
+
import { createHash, } from "node:crypto";
|
|
1
2
|
import * as fs from "node:fs";
|
|
2
3
|
import * as os from "node:os";
|
|
3
4
|
import * as path from "node:path";
|
|
5
|
+
import { fileURLToPath, } from "node:url";
|
|
4
6
|
const SKILL_URLS = [
|
|
5
7
|
new URL("../skills/dataiku-dss/SKILL.md", import.meta.url),
|
|
6
8
|
new URL("../../skills/dataiku-dss/SKILL.md", import.meta.url),
|
|
7
9
|
];
|
|
8
|
-
function
|
|
10
|
+
function skillDirectory() {
|
|
9
11
|
for (const skillUrl of SKILL_URLS) {
|
|
10
12
|
if (fs.existsSync(skillUrl))
|
|
11
|
-
return
|
|
13
|
+
return new URL("./", skillUrl);
|
|
12
14
|
}
|
|
13
15
|
throw new Error(`Bundled Dataiku skill not found. Checked: ${SKILL_URLS.map((url) => url.pathname).join(", ")}`);
|
|
14
16
|
}
|
|
17
|
+
function skillContent() {
|
|
18
|
+
return fs.readFileSync(new URL("SKILL.md", skillDirectory()), "utf-8");
|
|
19
|
+
}
|
|
15
20
|
export const AGENTS = {
|
|
16
21
|
claude: {
|
|
17
22
|
name: "Claude Code",
|
|
@@ -97,7 +102,39 @@ export function findWorkspaceRoot(startDir) {
|
|
|
97
102
|
}
|
|
98
103
|
return startDir;
|
|
99
104
|
}
|
|
100
|
-
|
|
105
|
+
function sha256Hex(value) {
|
|
106
|
+
return createHash("sha256").update(value).digest("hex");
|
|
107
|
+
}
|
|
108
|
+
function skillState(target, expectedSha256) {
|
|
109
|
+
if (!fs.existsSync(target))
|
|
110
|
+
return { status: "missing", };
|
|
111
|
+
const actualSha256 = sha256Hex(fs.readFileSync(target));
|
|
112
|
+
return {
|
|
113
|
+
status: actualSha256 === expectedSha256 ? "current" : "stale",
|
|
114
|
+
actualSha256,
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
/** Load the packaged tree, keeping portable relative paths in the report. */
|
|
118
|
+
function skillFiles() {
|
|
119
|
+
const root = fileURLToPath(skillDirectory());
|
|
120
|
+
const files = new Map();
|
|
121
|
+
function visit(relativeDir) {
|
|
122
|
+
const dir = path.join(root, relativeDir);
|
|
123
|
+
for (const entry of fs.readdirSync(dir, { withFileTypes: true, }).sort((a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0)) {
|
|
124
|
+
const relativePath = relativeDir ? `${relativeDir}/${entry.name}` : entry.name;
|
|
125
|
+
if (entry.isDirectory())
|
|
126
|
+
visit(relativePath);
|
|
127
|
+
else if (entry.isFile()) {
|
|
128
|
+
files.set(relativePath, fs.readFileSync(path.join(root, relativePath)));
|
|
129
|
+
}
|
|
130
|
+
else
|
|
131
|
+
throw new Error(`Unsupported bundled skill entry: ${relativePath}`);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
visit("");
|
|
135
|
+
return files;
|
|
136
|
+
}
|
|
137
|
+
function skillInstallPlan(agents, opts, bundle) {
|
|
101
138
|
const home = os.homedir();
|
|
102
139
|
const results = [];
|
|
103
140
|
for (const { id, def, via, } of agents) {
|
|
@@ -108,18 +145,65 @@ export function planSkillInstalls(agents, opts) {
|
|
|
108
145
|
: undefined;
|
|
109
146
|
if (!dir)
|
|
110
147
|
continue;
|
|
111
|
-
|
|
148
|
+
const files = [];
|
|
149
|
+
for (const [sourcePath, source,] of bundle) {
|
|
150
|
+
const relativePath = sourcePath === "SKILL.md" ? def.filename : sourcePath;
|
|
151
|
+
const target = path.join(dir, relativePath);
|
|
152
|
+
const expectedSha256 = sha256Hex(sourcePath === "SKILL.md" ? def.content() : source);
|
|
153
|
+
const state = skillState(target, expectedSha256);
|
|
154
|
+
files.push({
|
|
155
|
+
relativePath,
|
|
156
|
+
path: target,
|
|
157
|
+
...state,
|
|
158
|
+
changed: state.status !== "current",
|
|
159
|
+
expectedSha256,
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
const entrypoint = files.find((file) => file.relativePath === def.filename);
|
|
163
|
+
const changed = files.some((file) => file.changed);
|
|
164
|
+
results.push({
|
|
165
|
+
agent: id,
|
|
166
|
+
path: entrypoint.path,
|
|
167
|
+
via,
|
|
168
|
+
status: entrypoint.status === "missing" ? "missing" : changed ? "stale" : "current",
|
|
169
|
+
changed,
|
|
170
|
+
expectedSha256: entrypoint.expectedSha256,
|
|
171
|
+
...(entrypoint.actualSha256 !== undefined ? { actualSha256: entrypoint.actualSha256, } : {}),
|
|
172
|
+
files,
|
|
173
|
+
});
|
|
112
174
|
}
|
|
113
175
|
return results;
|
|
114
176
|
}
|
|
177
|
+
export function planSkillInstalls(agents, opts) {
|
|
178
|
+
return skillInstallPlan(agents, opts, skillFiles());
|
|
179
|
+
}
|
|
180
|
+
function writeSkillFile(file, content) {
|
|
181
|
+
if (!file.changed)
|
|
182
|
+
return;
|
|
183
|
+
const dir = path.dirname(file.path);
|
|
184
|
+
fs.mkdirSync(dir, { recursive: true, });
|
|
185
|
+
const tmpPath = path.join(dir, `.${path.basename(file.path)}.tmp-${process.pid}-${Date.now().toString(36)}`);
|
|
186
|
+
try {
|
|
187
|
+
fs.writeFileSync(tmpPath, content);
|
|
188
|
+
fs.renameSync(tmpPath, file.path);
|
|
189
|
+
}
|
|
190
|
+
finally {
|
|
191
|
+
fs.rmSync(tmpPath, { force: true, });
|
|
192
|
+
}
|
|
193
|
+
}
|
|
115
194
|
export function installSkill(agents, opts) {
|
|
116
|
-
const
|
|
195
|
+
const bundle = skillFiles();
|
|
196
|
+
const results = skillInstallPlan(agents, opts, bundle);
|
|
117
197
|
for (const result of results) {
|
|
118
|
-
|
|
119
|
-
if (!def)
|
|
198
|
+
if (!result.changed)
|
|
120
199
|
continue;
|
|
121
|
-
|
|
122
|
-
|
|
200
|
+
const def = agents.find((agent) => agent.id === result.agent).def;
|
|
201
|
+
// Publish referenced files first; never expose a new entrypoint with missing references.
|
|
202
|
+
for (const file of result.files) {
|
|
203
|
+
if (file.relativePath !== def.filename)
|
|
204
|
+
writeSkillFile(file, bundle.get(file.relativePath));
|
|
205
|
+
}
|
|
206
|
+
writeSkillFile(result.files.find((file) => file.relativePath === def.filename), def.content());
|
|
123
207
|
}
|
|
124
208
|
return results;
|
|
125
209
|
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import type { FlowZone } from "../schemas.js";
|
|
2
|
+
import type { NormalizedFlowMap, NormalizedFlowNode } from "./flow-map.js";
|
|
3
|
+
export type FlowRenderFormat = "ascii" | "mermaid";
|
|
4
|
+
export type FlowDiagnosticSeverity = "info" | "warning";
|
|
5
|
+
export interface AnalyzedFlowNode extends NormalizedFlowNode {
|
|
6
|
+
componentId: string;
|
|
7
|
+
layer: number;
|
|
8
|
+
zoneId: string;
|
|
9
|
+
zoneName: string;
|
|
10
|
+
}
|
|
11
|
+
export interface FlowMapZone {
|
|
12
|
+
id: string;
|
|
13
|
+
name: string;
|
|
14
|
+
color?: string;
|
|
15
|
+
position?: {
|
|
16
|
+
x: number;
|
|
17
|
+
y: number;
|
|
18
|
+
};
|
|
19
|
+
itemCount: number;
|
|
20
|
+
nodeIds: string[];
|
|
21
|
+
}
|
|
22
|
+
export interface FlowMapComponent {
|
|
23
|
+
id: string;
|
|
24
|
+
nodeIds: string[];
|
|
25
|
+
rootIds: string[];
|
|
26
|
+
leafIds: string[];
|
|
27
|
+
cyclic: boolean;
|
|
28
|
+
}
|
|
29
|
+
export interface FlowMapDiagnostic {
|
|
30
|
+
code: string;
|
|
31
|
+
severity: FlowDiagnosticSeverity;
|
|
32
|
+
message: string;
|
|
33
|
+
nodeIds?: string[];
|
|
34
|
+
componentIds?: string[];
|
|
35
|
+
edgeCount?: number;
|
|
36
|
+
}
|
|
37
|
+
export interface AnalyzedFlowMap extends Omit<NormalizedFlowMap, "nodes"> {
|
|
38
|
+
nodes: AnalyzedFlowNode[];
|
|
39
|
+
zones: FlowMapZone[];
|
|
40
|
+
components: FlowMapComponent[];
|
|
41
|
+
diagnostics: FlowMapDiagnostic[];
|
|
42
|
+
topologyFingerprint: string;
|
|
43
|
+
}
|
|
44
|
+
export interface FlowMapRendering {
|
|
45
|
+
format: FlowRenderFormat;
|
|
46
|
+
content: string;
|
|
47
|
+
}
|
|
48
|
+
export declare function flowTopologyFingerprint(map: NormalizedFlowMap): string;
|
|
49
|
+
export declare function analyzeFlowMap(map: NormalizedFlowMap, flowZones: FlowZone[], options?: {
|
|
50
|
+
topologyFingerprint?: string;
|
|
51
|
+
truncated?: boolean;
|
|
52
|
+
}): AnalyzedFlowMap;
|
|
53
|
+
export declare function renderFlowMap(map: AnalyzedFlowMap, format: FlowRenderFormat): FlowMapRendering;
|
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
import { compareStrings, stableHash, } from "./stable-hash.js";
|
|
2
|
+
function sorted(values) {
|
|
3
|
+
return [...values,].sort(compareStrings);
|
|
4
|
+
}
|
|
5
|
+
function validPosition(zone) {
|
|
6
|
+
const position = zone.position;
|
|
7
|
+
return position && Number.isFinite(position.x) && Number.isFinite(position.y)
|
|
8
|
+
? { x: position.x, y: position.y, }
|
|
9
|
+
: undefined;
|
|
10
|
+
}
|
|
11
|
+
function zoneItems(zone) {
|
|
12
|
+
return [...(zone.items ?? []), ...(zone.shared ?? []),];
|
|
13
|
+
}
|
|
14
|
+
function resolveZoneItemNodeId(item, projectKey, nodeIds) {
|
|
15
|
+
if (item.projectKey && item.projectKey !== projectKey) {
|
|
16
|
+
const qualified = `${item.projectKey}.${item.objectId}`;
|
|
17
|
+
if (nodeIds.has(qualified))
|
|
18
|
+
return qualified;
|
|
19
|
+
}
|
|
20
|
+
return nodeIds.has(item.objectId) ? item.objectId : undefined;
|
|
21
|
+
}
|
|
22
|
+
function flowAdjacency(map) {
|
|
23
|
+
const outgoingSets = new Map();
|
|
24
|
+
const undirectedSets = new Map();
|
|
25
|
+
const selfLoops = new Set();
|
|
26
|
+
for (const node of map.nodes) {
|
|
27
|
+
outgoingSets.set(node.id, new Set());
|
|
28
|
+
undirectedSets.set(node.id, new Set());
|
|
29
|
+
}
|
|
30
|
+
for (const edge of map.edges) {
|
|
31
|
+
if (!outgoingSets.has(edge.from) || !outgoingSets.has(edge.to))
|
|
32
|
+
continue;
|
|
33
|
+
outgoingSets.get(edge.from).add(edge.to);
|
|
34
|
+
undirectedSets.get(edge.from).add(edge.to);
|
|
35
|
+
undirectedSets.get(edge.to).add(edge.from);
|
|
36
|
+
if (edge.from === edge.to)
|
|
37
|
+
selfLoops.add(edge.from);
|
|
38
|
+
}
|
|
39
|
+
return {
|
|
40
|
+
outgoing: new Map([...outgoingSets,].map(([id, values,]) => [id, sorted(values),])),
|
|
41
|
+
undirected: new Map([...undirectedSets,].map(([id, values,]) => [id, sorted(values),])),
|
|
42
|
+
selfLoops,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
function strongComponents(nodeIds, outgoing, selfLoops) {
|
|
46
|
+
const incoming = new Map(nodeIds.map((nodeId) => [nodeId, [],]));
|
|
47
|
+
for (const [nodeId, successors,] of outgoing) {
|
|
48
|
+
for (const successor of successors)
|
|
49
|
+
incoming.get(successor)?.push(nodeId);
|
|
50
|
+
}
|
|
51
|
+
for (const predecessors of incoming.values())
|
|
52
|
+
predecessors.sort(compareStrings);
|
|
53
|
+
const visited = new Set();
|
|
54
|
+
const finished = [];
|
|
55
|
+
for (const start of nodeIds) {
|
|
56
|
+
if (visited.has(start))
|
|
57
|
+
continue;
|
|
58
|
+
visited.add(start);
|
|
59
|
+
const stack = [{ nodeId: start, next: 0, },];
|
|
60
|
+
while (stack.length > 0) {
|
|
61
|
+
const frame = stack[stack.length - 1];
|
|
62
|
+
const successors = outgoing.get(frame.nodeId) ?? [];
|
|
63
|
+
const successor = successors[frame.next];
|
|
64
|
+
if (successor !== undefined) {
|
|
65
|
+
frame.next += 1;
|
|
66
|
+
if (visited.has(successor))
|
|
67
|
+
continue;
|
|
68
|
+
visited.add(successor);
|
|
69
|
+
stack.push({ nodeId: successor, next: 0, });
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
finished.push(frame.nodeId);
|
|
73
|
+
stack.pop();
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
const assigned = new Set();
|
|
77
|
+
const components = [];
|
|
78
|
+
for (let index = finished.length - 1; index >= 0; index -= 1) {
|
|
79
|
+
const start = finished[index];
|
|
80
|
+
if (assigned.has(start))
|
|
81
|
+
continue;
|
|
82
|
+
const members = [];
|
|
83
|
+
const stack = [start,];
|
|
84
|
+
assigned.add(start);
|
|
85
|
+
while (stack.length > 0) {
|
|
86
|
+
const nodeId = stack.pop();
|
|
87
|
+
members.push(nodeId);
|
|
88
|
+
for (const predecessor of incoming.get(nodeId) ?? []) {
|
|
89
|
+
if (assigned.has(predecessor))
|
|
90
|
+
continue;
|
|
91
|
+
assigned.add(predecessor);
|
|
92
|
+
stack.push(predecessor);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
members.sort(compareStrings);
|
|
96
|
+
components.push({
|
|
97
|
+
id: -1,
|
|
98
|
+
nodeIds: members,
|
|
99
|
+
cyclic: members.length > 1 || selfLoops.has(members[0]),
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
components.sort((a, b) => compareStrings(a.nodeIds[0], b.nodeIds[0]));
|
|
103
|
+
components.forEach((component, index) => component.id = index);
|
|
104
|
+
return components;
|
|
105
|
+
}
|
|
106
|
+
function componentLayers(strong, edges) {
|
|
107
|
+
const strongByNode = new Map();
|
|
108
|
+
const outgoing = new Map();
|
|
109
|
+
const indegree = new Map();
|
|
110
|
+
const layers = new Map();
|
|
111
|
+
for (const component of strong) {
|
|
112
|
+
outgoing.set(component.id, new Set());
|
|
113
|
+
indegree.set(component.id, 0);
|
|
114
|
+
layers.set(component.id, 0);
|
|
115
|
+
for (const nodeId of component.nodeIds)
|
|
116
|
+
strongByNode.set(nodeId, component.id);
|
|
117
|
+
}
|
|
118
|
+
for (const edge of edges) {
|
|
119
|
+
const from = strongByNode.get(edge.from);
|
|
120
|
+
const to = strongByNode.get(edge.to);
|
|
121
|
+
if (from === undefined || to === undefined || from === to || outgoing.get(from).has(to)) {
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
outgoing.get(from).add(to);
|
|
125
|
+
indegree.set(to, indegree.get(to) + 1);
|
|
126
|
+
}
|
|
127
|
+
const ready = strong.filter((component) => indegree.get(component.id) === 0).map((component) => component.id);
|
|
128
|
+
while (ready.length > 0) {
|
|
129
|
+
const current = ready.pop();
|
|
130
|
+
for (const successor of outgoing.get(current)) {
|
|
131
|
+
layers.set(successor, Math.max(layers.get(successor), layers.get(current) + 1));
|
|
132
|
+
indegree.set(successor, indegree.get(successor) - 1);
|
|
133
|
+
if (indegree.get(successor) === 0)
|
|
134
|
+
ready.push(successor);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
return new Map([...strongByNode,].map(([nodeId, id,]) => [nodeId, layers.get(id) ?? 0,]));
|
|
138
|
+
}
|
|
139
|
+
function weakComponents(map, undirected, cyclicNodes) {
|
|
140
|
+
const seen = new Set();
|
|
141
|
+
const components = [];
|
|
142
|
+
const componentByNode = new Map();
|
|
143
|
+
const rootSet = new Set(map.roots);
|
|
144
|
+
const leafSet = new Set(map.leaves);
|
|
145
|
+
for (const start of map.nodes.map((node) => node.id)) {
|
|
146
|
+
if (seen.has(start))
|
|
147
|
+
continue;
|
|
148
|
+
const queue = [start,];
|
|
149
|
+
const nodeIds = [];
|
|
150
|
+
seen.add(start);
|
|
151
|
+
for (let index = 0; index < queue.length; index += 1) {
|
|
152
|
+
const current = queue[index];
|
|
153
|
+
nodeIds.push(current);
|
|
154
|
+
for (const adjacent of undirected.get(current) ?? []) {
|
|
155
|
+
if (seen.has(adjacent))
|
|
156
|
+
continue;
|
|
157
|
+
seen.add(adjacent);
|
|
158
|
+
queue.push(adjacent);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
nodeIds.sort(compareStrings);
|
|
162
|
+
const id = `component-${components.length + 1}`;
|
|
163
|
+
for (const nodeId of nodeIds)
|
|
164
|
+
componentByNode.set(nodeId, id);
|
|
165
|
+
components.push({
|
|
166
|
+
id,
|
|
167
|
+
nodeIds,
|
|
168
|
+
rootIds: nodeIds.filter((nodeId) => rootSet.has(nodeId)),
|
|
169
|
+
leafIds: nodeIds.filter((nodeId) => leafSet.has(nodeId)),
|
|
170
|
+
cyclic: nodeIds.some((nodeId) => cyclicNodes.has(nodeId)),
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
return { components, componentByNode, };
|
|
174
|
+
}
|
|
175
|
+
export function flowTopologyFingerprint(map) {
|
|
176
|
+
return stableHash({
|
|
177
|
+
nodes: [...map.nodes,]
|
|
178
|
+
.map((node) => ({ id: node.id, kind: node.kind, }))
|
|
179
|
+
.sort((a, b) => compareStrings(a.id, b.id)),
|
|
180
|
+
edges: [...map.edges,]
|
|
181
|
+
.map((edge) => ({ from: edge.from, to: edge.to, relation: edge.relation, }))
|
|
182
|
+
.sort((a, b) => compareStrings(a.from, b.from) || compareStrings(a.to, b.to)
|
|
183
|
+
|| compareStrings(a.relation, b.relation)),
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
export function analyzeFlowMap(map, flowZones, options = {}) {
|
|
187
|
+
const nodeIds = new Set(map.nodes.map((node) => node.id));
|
|
188
|
+
const diagnostics = [];
|
|
189
|
+
const sortedZones = [...flowZones,].sort((a, b) => {
|
|
190
|
+
if (a.id === "default")
|
|
191
|
+
return b.id === "default" ? 0 : 1;
|
|
192
|
+
if (b.id === "default")
|
|
193
|
+
return -1;
|
|
194
|
+
const ap = validPosition(a);
|
|
195
|
+
const bp = validPosition(b);
|
|
196
|
+
if (ap && bp)
|
|
197
|
+
return ap.x - bp.x || ap.y - bp.y || compareStrings(a.name, b.name);
|
|
198
|
+
if (ap)
|
|
199
|
+
return -1;
|
|
200
|
+
if (bp)
|
|
201
|
+
return 1;
|
|
202
|
+
return compareStrings(a.name, b.name) || compareStrings(a.id, b.id);
|
|
203
|
+
});
|
|
204
|
+
if (!sortedZones.some((zone) => zone.id === "default")) {
|
|
205
|
+
sortedZones.push({ id: "default", name: "Default", items: [], });
|
|
206
|
+
}
|
|
207
|
+
const zoneByNode = new Map();
|
|
208
|
+
for (const zone of sortedZones) {
|
|
209
|
+
for (const item of zoneItems(zone)) {
|
|
210
|
+
const nodeId = resolveZoneItemNodeId(item, map.projectKey, nodeIds);
|
|
211
|
+
if (!nodeId)
|
|
212
|
+
continue;
|
|
213
|
+
const previous = zoneByNode.get(nodeId);
|
|
214
|
+
if (previous && previous.id !== zone.id) {
|
|
215
|
+
diagnostics.push({
|
|
216
|
+
code: "duplicate_zone_assignment",
|
|
217
|
+
severity: "warning",
|
|
218
|
+
message: `Flow node ${nodeId} appears in zones ${previous.name} and ${zone.name}.`,
|
|
219
|
+
nodeIds: [nodeId,],
|
|
220
|
+
});
|
|
221
|
+
continue;
|
|
222
|
+
}
|
|
223
|
+
zoneByNode.set(nodeId, zone);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
const defaultZone = sortedZones.find((zone) => zone.id === "default");
|
|
227
|
+
for (const nodeId of nodeIds) {
|
|
228
|
+
if (!zoneByNode.has(nodeId))
|
|
229
|
+
zoneByNode.set(nodeId, defaultZone);
|
|
230
|
+
}
|
|
231
|
+
const adjacency = flowAdjacency(map);
|
|
232
|
+
const nodeIdList = sorted(nodeIds);
|
|
233
|
+
const strong = strongComponents(nodeIdList, adjacency.outgoing, adjacency.selfLoops);
|
|
234
|
+
const layers = componentLayers(strong, map.edges);
|
|
235
|
+
const cyclicNodes = new Set(strong.filter((component) => component.cyclic).flatMap((component) => component.nodeIds));
|
|
236
|
+
const { components, componentByNode, } = weakComponents(map, adjacency.undirected, cyclicNodes);
|
|
237
|
+
for (const component of strong.filter((candidate) => candidate.cyclic)) {
|
|
238
|
+
diagnostics.push({
|
|
239
|
+
code: "cycle_detected",
|
|
240
|
+
severity: "warning",
|
|
241
|
+
message: `Flow cycle contains ${component.nodeIds.length} node(s).`,
|
|
242
|
+
nodeIds: component.nodeIds,
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
if (components.length > 1) {
|
|
246
|
+
diagnostics.push({
|
|
247
|
+
code: "disconnected_components",
|
|
248
|
+
severity: "info",
|
|
249
|
+
message: `Flow contains ${components.length} disconnected components.`,
|
|
250
|
+
componentIds: components.map((component) => component.id),
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
const isolated = nodeIdList.filter((nodeId) => (adjacency.undirected.get(nodeId) ?? []).length === 0);
|
|
254
|
+
if (isolated.length > 0) {
|
|
255
|
+
diagnostics.push({
|
|
256
|
+
code: "isolated_nodes",
|
|
257
|
+
severity: "info",
|
|
258
|
+
message: `Flow contains ${isolated.length} isolated node(s).`,
|
|
259
|
+
nodeIds: isolated,
|
|
260
|
+
});
|
|
261
|
+
}
|
|
262
|
+
const defaultNodes = nodeIdList.filter((nodeId) => zoneByNode.get(nodeId).id === "default");
|
|
263
|
+
if (defaultNodes.length > 0) {
|
|
264
|
+
diagnostics.push({
|
|
265
|
+
code: "default_zone_items",
|
|
266
|
+
severity: "info",
|
|
267
|
+
message: `${defaultNodes.length} visible flow node(s) remain in the default zone.`,
|
|
268
|
+
nodeIds: defaultNodes,
|
|
269
|
+
});
|
|
270
|
+
}
|
|
271
|
+
const crossZoneEdges = map.edges.filter((edge) => {
|
|
272
|
+
const fromZone = zoneByNode.get(edge.from);
|
|
273
|
+
const toZone = zoneByNode.get(edge.to);
|
|
274
|
+
return fromZone !== undefined && toZone !== undefined && fromZone.id !== toZone.id;
|
|
275
|
+
}).length;
|
|
276
|
+
if (crossZoneEdges > 0) {
|
|
277
|
+
diagnostics.push({
|
|
278
|
+
code: "cross_zone_edges",
|
|
279
|
+
severity: "info",
|
|
280
|
+
message: `Flow contains ${crossZoneEdges} cross-zone edge(s).`,
|
|
281
|
+
edgeCount: crossZoneEdges,
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
if (options.truncated === true) {
|
|
285
|
+
diagnostics.push({
|
|
286
|
+
code: "analysis_truncated",
|
|
287
|
+
severity: "warning",
|
|
288
|
+
message: "Layers, components, zones, and diagnostics cover only the returned truncated map.",
|
|
289
|
+
});
|
|
290
|
+
}
|
|
291
|
+
diagnostics.sort((a, b) => compareStrings(a.code, b.code) || compareStrings(a.message, b.message));
|
|
292
|
+
const analyzedNodes = map.nodes.map((node) => {
|
|
293
|
+
const zone = zoneByNode.get(node.id);
|
|
294
|
+
return {
|
|
295
|
+
...node,
|
|
296
|
+
componentId: componentByNode.get(node.id),
|
|
297
|
+
layer: layers.get(node.id) ?? 0,
|
|
298
|
+
zoneId: zone.id,
|
|
299
|
+
zoneName: zone.name,
|
|
300
|
+
};
|
|
301
|
+
});
|
|
302
|
+
const visibleByZone = new Map();
|
|
303
|
+
for (const node of analyzedNodes) {
|
|
304
|
+
const ids = visibleByZone.get(node.zoneId) ?? [];
|
|
305
|
+
ids.push(node.id);
|
|
306
|
+
visibleByZone.set(node.zoneId, ids);
|
|
307
|
+
}
|
|
308
|
+
const zones = sortedZones.map((zone) => {
|
|
309
|
+
const position = validPosition(zone);
|
|
310
|
+
return {
|
|
311
|
+
id: zone.id,
|
|
312
|
+
name: zone.name,
|
|
313
|
+
...(zone.color ? { color: zone.color, } : {}),
|
|
314
|
+
...(position ? { position, } : {}),
|
|
315
|
+
itemCount: zoneItems(zone).length,
|
|
316
|
+
nodeIds: sorted(visibleByZone.get(zone.id) ?? []),
|
|
317
|
+
};
|
|
318
|
+
});
|
|
319
|
+
return {
|
|
320
|
+
...map,
|
|
321
|
+
nodes: analyzedNodes,
|
|
322
|
+
zones,
|
|
323
|
+
components,
|
|
324
|
+
diagnostics,
|
|
325
|
+
topologyFingerprint: options.topologyFingerprint ?? flowTopologyFingerprint(map),
|
|
326
|
+
};
|
|
327
|
+
}
|
|
328
|
+
function readableNode(node) {
|
|
329
|
+
return `${node.kind}:${node.name ?? node.id}`;
|
|
330
|
+
}
|
|
331
|
+
function mermaidLabel(value) {
|
|
332
|
+
return value.replaceAll("&", "&").replaceAll('"', """).replaceAll("<", "<")
|
|
333
|
+
.replaceAll(">", ">").replaceAll(/\s+/g, " ").trim();
|
|
334
|
+
}
|
|
335
|
+
function renderMermaid(map) {
|
|
336
|
+
const idByNode = new Map(map.nodes.map((node, index) => [node.id, `n${index}`,]));
|
|
337
|
+
const nodeById = new Map(map.nodes.map((node) => [node.id, node,]));
|
|
338
|
+
const lines = ["flowchart LR",];
|
|
339
|
+
for (const [zoneIndex, zone,] of map.zones.entries()) {
|
|
340
|
+
if (zone.nodeIds.length === 0)
|
|
341
|
+
continue;
|
|
342
|
+
lines.push(` subgraph z${zoneIndex}["${mermaidLabel(zone.name)}"]`);
|
|
343
|
+
for (const nodeId of zone.nodeIds) {
|
|
344
|
+
const node = nodeById.get(nodeId);
|
|
345
|
+
lines.push(` ${idByNode.get(nodeId)}["${mermaidLabel(readableNode(node))}"]`);
|
|
346
|
+
}
|
|
347
|
+
lines.push(" end");
|
|
348
|
+
}
|
|
349
|
+
for (const edge of map.edges) {
|
|
350
|
+
const from = idByNode.get(edge.from);
|
|
351
|
+
const to = idByNode.get(edge.to);
|
|
352
|
+
if (from && to)
|
|
353
|
+
lines.push(` ${from} --> ${to}`);
|
|
354
|
+
}
|
|
355
|
+
return lines.join("\n");
|
|
356
|
+
}
|
|
357
|
+
function renderAscii(map) {
|
|
358
|
+
const nodeById = new Map(map.nodes.map((node) => [node.id, node,]));
|
|
359
|
+
const lines = [];
|
|
360
|
+
for (const component of map.components) {
|
|
361
|
+
lines.push(`${component.id}${component.cyclic ? " (cyclic)" : ""}`);
|
|
362
|
+
const layers = new Map();
|
|
363
|
+
for (const nodeId of component.nodeIds) {
|
|
364
|
+
const node = nodeById.get(nodeId);
|
|
365
|
+
const entries = layers.get(node.layer) ?? [];
|
|
366
|
+
entries.push(`${readableNode(node)} [${node.zoneName}]`);
|
|
367
|
+
layers.set(node.layer, entries);
|
|
368
|
+
}
|
|
369
|
+
for (const [layer, entries,] of [...layers,].sort(([a,], [b,]) => a - b)) {
|
|
370
|
+
lines.push(` L${layer}: ${entries.sort(compareStrings).join(" | ")}`);
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
if (map.edges.length > 0) {
|
|
374
|
+
lines.push("edges");
|
|
375
|
+
for (const edge of map.edges)
|
|
376
|
+
lines.push(` ${edge.from} -> ${edge.to} (${edge.relation})`);
|
|
377
|
+
}
|
|
378
|
+
return lines.join("\n");
|
|
379
|
+
}
|
|
380
|
+
export function renderFlowMap(map, format) {
|
|
381
|
+
return { format, content: format === "mermaid" ? renderMermaid(map) : renderAscii(map), };
|
|
382
|
+
}
|
|
@@ -1,3 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Safety budgets applied by {@link inspectProjectArchive} while reading a
|
|
3
|
+
* project archive. A small malicious archive must not be able to exhaust
|
|
4
|
+
* CPU or memory during the pre-upload local inspection, so every limit is
|
|
5
|
+
* enforced: members whose central-directory record declares a compressed
|
|
6
|
+
* size or expanded/compressed ratio above budget are never streamed,
|
|
7
|
+
* per-member and aggregate expanded-byte overruns destroy the member
|
|
8
|
+
* stream immediately, and archives with too many members are reported
|
|
9
|
+
* without opening any member stream. All limits are per-inspection
|
|
10
|
+
* configuration: pass a partial object as `overrides` to adjust only what
|
|
11
|
+
* you need, otherwise the exported defaults apply.
|
|
12
|
+
*/
|
|
13
|
+
export interface ProjectArchiveInspectionLimits {
|
|
14
|
+
/** Maximum number of central-directory members. Default: 100000. */
|
|
15
|
+
maxMembers: number;
|
|
16
|
+
/** Maximum declared compressed bytes of one member. Default: 4 GiB. */
|
|
17
|
+
maxCompressedBytesPerMember: number;
|
|
18
|
+
/** Maximum expanded bytes streamed from one member. Default: 4 GiB. */
|
|
19
|
+
maxExpandedBytesPerMember: number;
|
|
20
|
+
/**
|
|
21
|
+
* Maximum declared expanded-to-compressed ratio of one member (zip-bomb
|
|
22
|
+
* guard, computed from the central-directory record before streaming).
|
|
23
|
+
* Default: 1000. Legitimate archives with members compressing better
|
|
24
|
+
* than this need a higher limit to be inspected.
|
|
25
|
+
*/
|
|
26
|
+
maxCompressionRatio: number;
|
|
27
|
+
/** Maximum total expanded bytes across all streamed members. Default: 16 GiB. */
|
|
28
|
+
maxTotalExpandedBytes: number;
|
|
29
|
+
}
|
|
30
|
+
export declare const PROJECT_ARCHIVE_DEFAULT_LIMITS: Readonly<ProjectArchiveInspectionLimits>;
|
|
1
31
|
export type ProjectArchiveIssueSeverity = "error" | "warning";
|
|
2
32
|
export interface ProjectArchiveIssue {
|
|
3
33
|
severity: ProjectArchiveIssueSeverity;
|
|
@@ -22,11 +52,20 @@ export interface ProjectArchiveInspection {
|
|
|
22
52
|
* the generic dangling-reference/orphan-member graph checks. Identity,
|
|
23
53
|
* history and machine-specific residue is reported only as warnings carrying
|
|
24
54
|
* member paths and pattern counts, never matched values.
|
|
55
|
+
*
|
|
56
|
+
* Safety budgets (see {@link ProjectArchiveInspectionLimits}) bound every
|
|
57
|
+
* amount of work: archives with more members than `maxMembers` are reported
|
|
58
|
+
* without opening any member stream, members whose central-directory record
|
|
59
|
+
* declares a compressed size or compression ratio above budget are skipped
|
|
60
|
+
* without being read, and per-member/aggregate expanded-byte overruns
|
|
61
|
+
* destroy the member stream immediately. Pass a partial limits object as
|
|
62
|
+
* `overrides` to customize the budgets for a single inspection; importers
|
|
63
|
+
* use the exported defaults.
|
|
25
64
|
*/
|
|
26
|
-
export declare function inspectProjectArchive(filePath: string): Promise<ProjectArchiveInspection>;
|
|
65
|
+
export declare function inspectProjectArchive(filePath: string, overrides?: Partial<ProjectArchiveInspectionLimits>): Promise<ProjectArchiveInspection>;
|
|
27
66
|
/**
|
|
28
67
|
* Like `inspectProjectArchive`, but throws a `ClientValidationError` with the
|
|
29
68
|
* `validation_failed` code when the inspection contains any error-severity
|
|
30
69
|
* issue, so the archive cannot be imported.
|
|
31
70
|
*/
|
|
32
|
-
export declare function assertImportableProjectArchive(filePath: string): Promise<ProjectArchiveInspection>;
|
|
71
|
+
export declare function assertImportableProjectArchive(filePath: string, overrides?: Partial<ProjectArchiveInspectionLimits>): Promise<ProjectArchiveInspection>;
|