@dataparade/cli 0.0.3 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -0
- package/dist/package.json +85 -0
- package/dist/src/ai-enrichment/agent-orchestrator.d.ts +2 -0
- package/dist/src/ai-enrichment/agent-orchestrator.js +8 -5
- package/dist/src/ai-enrichment/providers/hosted-worker-infer-provider.d.ts +10 -0
- package/dist/src/ai-enrichment/providers/hosted-worker-infer-provider.js +47 -0
- package/dist/src/cli.js +88 -1
- package/dist/src/config/env.js +2 -0
- package/dist/src/config/resolve.js +3 -0
- package/dist/src/config/scan-env.d.ts +1 -0
- package/dist/src/config/scan-env.js +5 -0
- package/dist/src/config/types.d.ts +1 -0
- package/dist/src/config/upload-env.d.ts +1 -0
- package/dist/src/config/upload-env.js +10 -0
- package/dist/src/config/validate-scan-ai.d.ts +1 -1
- package/dist/src/config/validate-scan-ai.js +5 -0
- package/dist/src/core/pipeline/ai-orchestrator-options.js +5 -0
- package/dist/src/core/schema/scan-config.schema.d.ts +2 -0
- package/dist/src/core/schema/scan-config.schema.js +2 -1
- package/dist/src/core/types/config.d.ts +3 -1
- package/dist/src/observability/scan-sentry.d.ts +1 -1
- package/dist/src/output/json.d.ts +4 -0
- package/dist/src/output/json.js +7 -2
- package/dist/src/platform-api/dataparade-app-base-url.d.ts +7 -0
- package/dist/src/platform-api/dataparade-app-base-url.js +14 -0
- package/dist/src/platform-api/upload-client.d.ts +3 -0
- package/dist/src/platform-api/upload-client.js +38 -0
- package/dist/src/platform-api/upload.types.d.ts +10 -0
- package/dist/src/platform-api/upload.types.js +2 -0
- package/dist/src/upload/build-preview-url.d.ts +1 -0
- package/dist/src/upload/build-preview-url.js +10 -0
- package/dist/src/upload/run-upload.d.ts +2 -0
- package/dist/src/upload/run-upload.js +18 -0
- package/dist/src/upload/types.d.ts +11 -0
- package/dist/src/upload/types.js +2 -0
- package/dist/tests/unit/cli/scan-quota-flow.spec.js +2 -0
- package/dist/tests/unit/config/upload-env.spec.d.ts +1 -0
- package/dist/tests/unit/config/upload-env.spec.js +12 -0
- package/dist/tests/unit/config/validate-scan-ai.spec.js +8 -0
- package/dist/tests/unit/core/output-json.spec.js +4 -1
- package/dist/tests/unit/platform-api/dataparade-app-base-url.spec.d.ts +1 -0
- package/dist/tests/unit/platform-api/dataparade-app-base-url.spec.js +13 -0
- package/dist/tests/unit/platform-api/upload-client.spec.d.ts +1 -0
- package/dist/tests/unit/platform-api/upload-client.spec.js +40 -0
- package/dist/tests/unit/publish-manifest.spec.d.ts +1 -0
- package/dist/tests/unit/publish-manifest.spec.js +18 -0
- package/dist/tests/unit/upload/build-preview-url.spec.d.ts +1 -0
- package/dist/tests/unit/upload/build-preview-url.spec.js +19 -0
- package/package.json +4 -3
- package/patterns/README.md +322 -0
- package/patterns/actor.patterns.yaml +51 -0
- package/patterns/aws-terraform-catalog.snapshot.json +1760 -0
- package/patterns/aws-terraform-service-hints.generated.json +1446 -0
- package/patterns/azure-terraform-catalog.snapshot.json +1195 -0
- package/patterns/azure-terraform-service-hints.generated.json +864 -0
- package/patterns/classifier/actors.classifier.yaml +17 -0
- package/patterns/classifier/components.classifier.yaml +110 -0
- package/patterns/classifier/third-party.classifier.yaml +169 -0
- package/patterns/kubernetes-terraform-catalog.snapshot.json +94 -0
- package/patterns/kubernetes-terraform-service-hints.generated.json +258 -0
- package/patterns/non-pii-signals.rules.yaml +84 -0
- package/patterns/pii-signals.rules.yaml +115 -0
- package/patterns/property.patterns.yaml +431 -0
- package/patterns/provider-topology.rules.yaml +765 -0
- package/patterns/python.md +211 -0
- package/patterns/python.patterns.yaml +160 -0
- package/patterns/terraform.md +37 -0
- package/patterns/terraform.patterns.yaml +495 -0
- package/patterns/third-party.patterns.yaml +167 -0
- package/patterns/typescript.patterns.yaml +139 -0
package/README.md
CHANGED
|
@@ -86,6 +86,30 @@ node dist/bin/cli.js scan path/to/your/project
|
|
|
86
86
|
|
|
87
87
|
---
|
|
88
88
|
|
|
89
|
+
## Upload to dashboard
|
|
90
|
+
|
|
91
|
+
After a scan, the CLI **auto-uploads** `dataflow.json` when `DATAPARADE_WORKSPACE_API_KEY` is set (from **Workspace → Access keys**). This creates an **import preview draft** in the web app — not a finished assessment. Open the printed URL to review and edit before creating the diagram.
|
|
92
|
+
|
|
93
|
+
- Opt out: `--skip-auto-upload` or `DATAPARADE_SKIP_AUTO_UPLOAD=true`
|
|
94
|
+
- Without a workspace key, the scan still succeeds; the CLI suggests running `upload` later
|
|
95
|
+
- Upload alone does **not** consume scan quota (platform AI scans still use preflight/complete as before)
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
# Upload an existing file
|
|
99
|
+
node dist/bin/cli.js upload ./dataflow.json --project-name "My service"
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
Environment:
|
|
103
|
+
|
|
104
|
+
| Variable | Purpose |
|
|
105
|
+
|----------|---------|
|
|
106
|
+
| `DATAPARADE_WORKSPACE_API_KEY` | Auth for upload (same key as platform AI) |
|
|
107
|
+
| `DATAPARADE_API_BASE_URL` | Backend API (default: production AWS) |
|
|
108
|
+
| `DATAPARADE_APP_URL` | Frontend base for preview links (default: production app) |
|
|
109
|
+
| `DATAPARADE_SKIP_AUTO_UPLOAD` | Skip post-scan upload when `true`/`1` |
|
|
110
|
+
|
|
111
|
+
---
|
|
112
|
+
|
|
89
113
|
## Output: `dataflow.json`
|
|
90
114
|
|
|
91
115
|
By default (no `--output` flag), the CLI writes `./dataflow.json` in the current working directory.
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@dataparade/cli",
|
|
3
|
+
"version": "0.0.5",
|
|
4
|
+
"description": "dataPARADE CLI — scan codebases for privacy and security data flows",
|
|
5
|
+
"publishConfig": {
|
|
6
|
+
"access": "public",
|
|
7
|
+
"registry": "https://registry.npmjs.org/"
|
|
8
|
+
},
|
|
9
|
+
"license": "GPL-3.0-or-later",
|
|
10
|
+
"author": "DataParade",
|
|
11
|
+
"homepage": "https://github.com/DataParade-io/dataparade-cli#readme",
|
|
12
|
+
"repository": {
|
|
13
|
+
"type": "git",
|
|
14
|
+
"url": "git+https://github.com/DataParade-io/dataparade-cli.git"
|
|
15
|
+
},
|
|
16
|
+
"bugs": {
|
|
17
|
+
"url": "https://github.com/DataParade-io/dataparade-cli/issues"
|
|
18
|
+
},
|
|
19
|
+
"engines": {
|
|
20
|
+
"node": ">=20"
|
|
21
|
+
},
|
|
22
|
+
"keywords": [
|
|
23
|
+
"dataparade",
|
|
24
|
+
"privacy",
|
|
25
|
+
"data-flow",
|
|
26
|
+
"security",
|
|
27
|
+
"cli",
|
|
28
|
+
"scanner"
|
|
29
|
+
],
|
|
30
|
+
"private": false,
|
|
31
|
+
"main": "dist/src/index.js",
|
|
32
|
+
"types": "dist/src/index.d.ts",
|
|
33
|
+
"exports": {
|
|
34
|
+
".": {
|
|
35
|
+
"types": "./dist/src/index.d.ts",
|
|
36
|
+
"default": "./dist/src/index.js"
|
|
37
|
+
},
|
|
38
|
+
"./ai-providers": {
|
|
39
|
+
"types": "./dist/src/ai-enrichment/providers/index.d.ts",
|
|
40
|
+
"default": "./dist/src/ai-enrichment/providers/index.js"
|
|
41
|
+
}
|
|
42
|
+
},
|
|
43
|
+
"bin": {
|
|
44
|
+
"@dataparade/cli": "dist/bin/cli.js"
|
|
45
|
+
},
|
|
46
|
+
"scripts": {
|
|
47
|
+
"generate:terraform-provider-hints": "node scripts/generate-terraform-provider-hints.mjs",
|
|
48
|
+
"generate:aws-terraform-hints": "node scripts/generate-aws-terraform-hints.mjs",
|
|
49
|
+
"generate:azure-terraform-hints": "node scripts/generate-terraform-provider-hints.mjs --azure-only",
|
|
50
|
+
"generate:kubernetes-terraform-hints": "node scripts/generate-terraform-provider-hints.mjs --kubernetes-only",
|
|
51
|
+
"build": "pnpm exec tsc -p tsconfig.json",
|
|
52
|
+
"prepublishOnly": "pnpm run build && pnpm run test",
|
|
53
|
+
"lint": "eslint \"src/**/*.ts\" \"tests/**/*.ts\"",
|
|
54
|
+
"pretest": "pnpm run build",
|
|
55
|
+
"test": "jest",
|
|
56
|
+
"scan": "node dist/bin/cli.js scan",
|
|
57
|
+
"eval:models": "pnpm exec node dist/src/evals/run-model-matrix.js"
|
|
58
|
+
},
|
|
59
|
+
"dependencies": {
|
|
60
|
+
"@sentry/node": "^10.38.0",
|
|
61
|
+
"commander": "^12.1.0",
|
|
62
|
+
"dotenv": "^16.4.7",
|
|
63
|
+
"langsmith": "^0.3.14",
|
|
64
|
+
"typescript": "^5.7.3",
|
|
65
|
+
"yaml": "^2.8.2",
|
|
66
|
+
"zod": "^4.3.5"
|
|
67
|
+
},
|
|
68
|
+
"devDependencies": {
|
|
69
|
+
"@eslint/js": "^9.18.0",
|
|
70
|
+
"@types/jest": "^29.5.14",
|
|
71
|
+
"@types/node": "^22.0.0",
|
|
72
|
+
"eslint": "^9.18.0",
|
|
73
|
+
"globals": "^15.14.0",
|
|
74
|
+
"jest": "^29.7.0",
|
|
75
|
+
"ts-jest": "^29.2.5",
|
|
76
|
+
"typescript-eslint": "^8.20.0"
|
|
77
|
+
},
|
|
78
|
+
"files": [
|
|
79
|
+
"dist",
|
|
80
|
+
"patterns",
|
|
81
|
+
"LICENSE",
|
|
82
|
+
"README.md",
|
|
83
|
+
"package.json"
|
|
84
|
+
]
|
|
85
|
+
}
|
|
@@ -18,6 +18,8 @@ export interface AgentOrchestratorOptions {
|
|
|
18
18
|
skipStructuralHeuristics?: boolean;
|
|
19
19
|
/** Platform-billed LLM via DataParade API proxy. */
|
|
20
20
|
platformProxy?: PlatformProxyProviderConfig;
|
|
21
|
+
/** Hosted scan in VPC worker: loopback proxy → scan-cli-ai-helper Lambda. */
|
|
22
|
+
hostedWorkerInferProxyUrl?: string;
|
|
21
23
|
}
|
|
22
24
|
export declare function generateAgenticProposals(input: RunInferencePipelineInput, plan: InferencePlannerResult, candidates: AiInferenceCandidate[], options: AgentOrchestratorOptions): Promise<{
|
|
23
25
|
proposals: Array<{
|
|
@@ -10,6 +10,7 @@ const third_party_evidence_1 = require("./third-party-evidence");
|
|
|
10
10
|
const provider_contract_1 = require("./providers/provider-contract");
|
|
11
11
|
const scan_paths_1 = require("./scan-paths");
|
|
12
12
|
const platform_proxy_provider_1 = require("./providers/platform-proxy-provider");
|
|
13
|
+
const hosted_worker_infer_provider_1 = require("./providers/hosted-worker-infer-provider");
|
|
13
14
|
const PROPERTY_AGENT_SLICE_SIZE = 12;
|
|
14
15
|
const TP_AGENT_MAX_ROUNDS = 3;
|
|
15
16
|
const TP_AGENT_MAX_FILES = 8;
|
|
@@ -156,11 +157,13 @@ async function generateAgenticProposals(input, plan, candidates, options) {
|
|
|
156
157
|
const skipStructural = options.skipStructuralHeuristics === true;
|
|
157
158
|
const provider = options.platformProxy
|
|
158
159
|
? new platform_proxy_provider_1.PlatformProxyProvider(options.platformProxy)
|
|
159
|
-
: (
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
160
|
+
: options.hostedWorkerInferProxyUrl?.trim()
|
|
161
|
+
? new hosted_worker_infer_provider_1.HostedWorkerInferProvider(options.hostedWorkerInferProxyUrl.trim())
|
|
162
|
+
: (0, providers_1.createAiProvider)(options.provider, {
|
|
163
|
+
endpoint: options.endpoint,
|
|
164
|
+
model: options.model,
|
|
165
|
+
apiKey: options.apiKey,
|
|
166
|
+
});
|
|
164
167
|
const generated = [];
|
|
165
168
|
const agenticTrace = [];
|
|
166
169
|
const aggregateUsage = {
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { AiProvider, AiProviderRequest, AiProviderResult } from "./types";
|
|
2
|
+
/**
|
|
3
|
+
* Sync infer via the scan worker's loopback proxy (VPC → scan-cli-ai-helper Lambda).
|
|
4
|
+
*/
|
|
5
|
+
export declare class HostedWorkerInferProvider implements AiProvider {
|
|
6
|
+
private readonly inferUrl;
|
|
7
|
+
readonly id: "openai";
|
|
8
|
+
constructor(inferUrl: string);
|
|
9
|
+
infer(request: AiProviderRequest): Promise<AiProviderResult>;
|
|
10
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.HostedWorkerInferProvider = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* Sync infer via the scan worker's loopback proxy (VPC → scan-cli-ai-helper Lambda).
|
|
6
|
+
*/
|
|
7
|
+
class HostedWorkerInferProvider {
|
|
8
|
+
constructor(inferUrl) {
|
|
9
|
+
this.inferUrl = inferUrl;
|
|
10
|
+
this.id = "openai";
|
|
11
|
+
}
|
|
12
|
+
async infer(request) {
|
|
13
|
+
const res = await fetch(this.inferUrl, {
|
|
14
|
+
method: "POST",
|
|
15
|
+
headers: { "Content-Type": "application/json" },
|
|
16
|
+
body: JSON.stringify({
|
|
17
|
+
prompt: request.prompt,
|
|
18
|
+
maxTokens: request.maxTokens,
|
|
19
|
+
temperature: request.temperature,
|
|
20
|
+
}),
|
|
21
|
+
});
|
|
22
|
+
let body;
|
|
23
|
+
try {
|
|
24
|
+
body = (await res.json());
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
throw new Error(`Hosted scan AI infer failed (${res.status})`);
|
|
28
|
+
}
|
|
29
|
+
if (!res.ok) {
|
|
30
|
+
throw new Error(body.error?.trim() ||
|
|
31
|
+
`Hosted scan AI infer failed (${res.status})`);
|
|
32
|
+
}
|
|
33
|
+
const proposals = (Array.isArray(body.proposals) ? body.proposals : []);
|
|
34
|
+
const usage = body.usage;
|
|
35
|
+
return {
|
|
36
|
+
proposals,
|
|
37
|
+
usage: usage && typeof usage.totalTokens === "number"
|
|
38
|
+
? {
|
|
39
|
+
inputTokens: Math.max(0, usage.inputTokens ?? 0),
|
|
40
|
+
outputTokens: Math.max(0, usage.outputTokens ?? 0),
|
|
41
|
+
totalTokens: Math.max(0, usage.totalTokens),
|
|
42
|
+
}
|
|
43
|
+
: undefined,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
exports.HostedWorkerInferProvider = HostedWorkerInferProvider;
|
package/dist/src/cli.js
CHANGED
|
@@ -40,7 +40,9 @@ exports.run = run;
|
|
|
40
40
|
require("./config/load-cli-env");
|
|
41
41
|
const path_1 = __importDefault(require("path"));
|
|
42
42
|
const commander_1 = require("commander");
|
|
43
|
+
const package_json_1 = __importDefault(require("../package.json"));
|
|
43
44
|
const json_1 = require("./output/json");
|
|
45
|
+
const upload_env_1 = require("./config/upload-env");
|
|
44
46
|
const types_1 = require("./ai-enrichment/types");
|
|
45
47
|
const inference_scope_1 = require("./config/inference-scope");
|
|
46
48
|
const redact_1 = require("./config/redact");
|
|
@@ -112,7 +114,7 @@ function createProgram() {
|
|
|
112
114
|
program
|
|
113
115
|
.name("dataparade")
|
|
114
116
|
.description("DataParade CLI - scan codebases for data flow components")
|
|
115
|
-
.version(
|
|
117
|
+
.version(package_json_1.default.version);
|
|
116
118
|
program
|
|
117
119
|
.command("scan <path>")
|
|
118
120
|
.description("Scan a directory (or a single supported source file) for data flow components")
|
|
@@ -143,6 +145,7 @@ function createProgram() {
|
|
|
143
145
|
.option("--api-key <key>", "(deprecated) alias for --workspace-api-key")
|
|
144
146
|
.option("--byok-provider <provider>", `BYOK LLM provider when using your own API key: ${types_1.AI_PROVIDER_IDS.join("|")}`)
|
|
145
147
|
.option("--byok-model <model>", "BYOK model name (env: SCAN_BYOK_MODEL)")
|
|
148
|
+
.option("--skip-auto-upload", "Do not upload dataflow.json to the dashboard after scan (env: DATAPARADE_SKIP_AUTO_UPLOAD)")
|
|
146
149
|
.action(async (path, options) => {
|
|
147
150
|
let cliQuotaJobId;
|
|
148
151
|
let platformQuotaApiKey;
|
|
@@ -368,16 +371,48 @@ function createProgram() {
|
|
|
368
371
|
}
|
|
369
372
|
if (diagramGraph) {
|
|
370
373
|
const dataflowOutputPath = path_1.default.resolve(process.cwd(), options.output ?? "dataflow.json");
|
|
374
|
+
const resolvedProjectName = config.projectName?.trim() ||
|
|
375
|
+
path_1.default.basename(scanEntry.scanRootDir);
|
|
371
376
|
try {
|
|
372
377
|
(0, json_1.writeDataflowJson)({
|
|
373
378
|
scanResult,
|
|
374
379
|
graph: diagramGraph,
|
|
375
380
|
outputPath: dataflowOutputPath,
|
|
381
|
+
projectName: resolvedProjectName,
|
|
376
382
|
});
|
|
377
383
|
// Always print a short message so non-interactive callers and
|
|
378
384
|
// tests can rely on it.
|
|
379
385
|
// eslint-disable-next-line no-console
|
|
380
386
|
console.log(`[scan] dataflow.json written to ${dataflowOutputPath}`);
|
|
387
|
+
const skipAutoUpload = Boolean(options.skipAutoUpload) ||
|
|
388
|
+
(0, upload_env_1.resolveSkipAutoUpload)(process.env);
|
|
389
|
+
if (!skipAutoUpload) {
|
|
390
|
+
const uploadApiKey = workspaceApiKey;
|
|
391
|
+
if (!uploadApiKey) {
|
|
392
|
+
// eslint-disable-next-line no-console
|
|
393
|
+
console.log(`[scan] Auto-upload skipped (no workspace API key). Run: dataparade upload ${dataflowOutputPath}`);
|
|
394
|
+
}
|
|
395
|
+
else {
|
|
396
|
+
try {
|
|
397
|
+
const { runDataflowUpload } = await Promise.resolve().then(() => __importStar(require("./upload/run-upload")));
|
|
398
|
+
const dataflowWrapper = (0, json_1.buildDataflowWrapper)(scanResult, diagramGraph, { projectName: resolvedProjectName });
|
|
399
|
+
await runDataflowUpload({
|
|
400
|
+
apiKey: uploadApiKey,
|
|
401
|
+
dataflow: dataflowWrapper,
|
|
402
|
+
projectName: resolvedProjectName,
|
|
403
|
+
scanJobId: cliQuotaJobId,
|
|
404
|
+
logPrefix: "[scan]",
|
|
405
|
+
});
|
|
406
|
+
}
|
|
407
|
+
catch (uploadError) {
|
|
408
|
+
const uploadMessage = uploadError instanceof Error
|
|
409
|
+
? uploadError.message
|
|
410
|
+
: "Unknown upload error.";
|
|
411
|
+
// eslint-disable-next-line no-console
|
|
412
|
+
console.error(`[scan] Auto-upload failed: ${uploadMessage}`);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
}
|
|
381
416
|
}
|
|
382
417
|
catch (dataflowError) {
|
|
383
418
|
await (0, scan_sentry_1.reportScanCliError)({
|
|
@@ -453,6 +488,58 @@ function createProgram() {
|
|
|
453
488
|
}
|
|
454
489
|
}
|
|
455
490
|
});
|
|
491
|
+
program
|
|
492
|
+
.command("upload <file>")
|
|
493
|
+
.description("Upload a dataflow.json file to the dashboard as an import preview")
|
|
494
|
+
.option("--project-name <name>", "Assessment name shown in the preview (default: from file metadata if present)")
|
|
495
|
+
.option("--workspace-api-key <key>", "DataParade workspace API key (env: DATAPARADE_WORKSPACE_API_KEY)")
|
|
496
|
+
.option("--api-key <key>", "(deprecated) alias for --workspace-api-key")
|
|
497
|
+
.action(async (file, options) => {
|
|
498
|
+
const apiKey = options.workspaceApiKey?.trim() ||
|
|
499
|
+
options.apiKey?.trim() ||
|
|
500
|
+
(0, scan_env_1.resolveWorkspaceApiKey)(process.env);
|
|
501
|
+
if (!apiKey) {
|
|
502
|
+
// eslint-disable-next-line no-console
|
|
503
|
+
console.error("[upload] error: workspace API key is required (DATAPARADE_WORKSPACE_API_KEY or --workspace-api-key)");
|
|
504
|
+
process.exitCode = 1;
|
|
505
|
+
return;
|
|
506
|
+
}
|
|
507
|
+
const filePath = path_1.default.resolve(process.cwd(), file || "dataflow.json");
|
|
508
|
+
let dataflow;
|
|
509
|
+
try {
|
|
510
|
+
const { readFileSync } = await Promise.resolve().then(() => __importStar(require("fs")));
|
|
511
|
+
dataflow = JSON.parse(readFileSync(filePath, "utf8"));
|
|
512
|
+
}
|
|
513
|
+
catch (err) {
|
|
514
|
+
const message = err instanceof Error ? err.message : "Failed to read dataflow.json";
|
|
515
|
+
// eslint-disable-next-line no-console
|
|
516
|
+
console.error(`[upload] error: ${message}`);
|
|
517
|
+
process.exitCode = 1;
|
|
518
|
+
return;
|
|
519
|
+
}
|
|
520
|
+
let projectName = options.projectName?.trim();
|
|
521
|
+
if (!projectName && dataflow && typeof dataflow === "object") {
|
|
522
|
+
const meta = dataflow
|
|
523
|
+
.metadata;
|
|
524
|
+
if (typeof meta?.projectName === "string" && meta.projectName.trim()) {
|
|
525
|
+
projectName = meta.projectName.trim();
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
try {
|
|
529
|
+
const { runDataflowUpload } = await Promise.resolve().then(() => __importStar(require("./upload/run-upload")));
|
|
530
|
+
await runDataflowUpload({
|
|
531
|
+
apiKey,
|
|
532
|
+
dataflow,
|
|
533
|
+
projectName,
|
|
534
|
+
});
|
|
535
|
+
}
|
|
536
|
+
catch (err) {
|
|
537
|
+
const message = err instanceof Error ? err.message : "Upload failed.";
|
|
538
|
+
// eslint-disable-next-line no-console
|
|
539
|
+
console.error(`[upload] error: ${message}`);
|
|
540
|
+
process.exitCode = 1;
|
|
541
|
+
}
|
|
542
|
+
});
|
|
456
543
|
program
|
|
457
544
|
.command("config")
|
|
458
545
|
.description("View the effective configuration for the current project")
|
package/dist/src/config/env.js
CHANGED
|
@@ -50,6 +50,7 @@ function loadCliConfigEnv(env) {
|
|
|
50
50
|
const aiToolLoopMaxSearches = parseNumber(env.SCAN_AI_TOOL_LOOP_MAX_SEARCHES);
|
|
51
51
|
const aiThirdPartyDataFlowEnabled = parseBoolean(env.SCAN_AI_THIRD_PARTY_DATA_FLOW);
|
|
52
52
|
const workspaceApiKey = (0, scan_env_1.resolveWorkspaceApiKey)(env);
|
|
53
|
+
const hostedInferProxyUrl = (0, scan_env_1.resolveHostedInferProxyUrl)(env);
|
|
53
54
|
return {
|
|
54
55
|
excludePaths,
|
|
55
56
|
minimumConfidence,
|
|
@@ -71,5 +72,6 @@ function loadCliConfigEnv(env) {
|
|
|
71
72
|
aiToolLoopMaxSearches,
|
|
72
73
|
aiThirdPartyDataFlowEnabled,
|
|
73
74
|
workspaceApiKey: workspaceApiKey || undefined,
|
|
75
|
+
hostedInferProxyUrl: hostedInferProxyUrl || undefined,
|
|
74
76
|
};
|
|
75
77
|
}
|
|
@@ -283,6 +283,9 @@ function resolveScanConfiguration(options) {
|
|
|
283
283
|
if (envConfig.workspaceApiKey) {
|
|
284
284
|
overrides.workspaceApiKey = envConfig.workspaceApiKey;
|
|
285
285
|
}
|
|
286
|
+
if (envConfig.hostedInferProxyUrl) {
|
|
287
|
+
overrides.hostedInferProxyUrl = envConfig.hostedInferProxyUrl;
|
|
288
|
+
}
|
|
286
289
|
if (envConfig.aiModel) {
|
|
287
290
|
overrides.aiModel = envConfig.aiModel;
|
|
288
291
|
}
|
|
@@ -3,5 +3,6 @@ export declare function resolveByokApiKey(env: NodeJS.ProcessEnv): string | unde
|
|
|
3
3
|
export declare function resolveByokProvider(env: NodeJS.ProcessEnv): string | undefined;
|
|
4
4
|
export declare function resolveByokModel(env: NodeJS.ProcessEnv): string | undefined;
|
|
5
5
|
export declare function resolveScanAiInference(env: NodeJS.ProcessEnv): string | undefined;
|
|
6
|
+
export declare function resolveHostedInferProxyUrl(env: NodeJS.ProcessEnv): string | undefined;
|
|
6
7
|
/** Extra provider HTTP / normalization logging (all providers, not OpenAI-only). */
|
|
7
8
|
export declare function isScanAiDebugEnabled(env?: NodeJS.ProcessEnv): boolean;
|
|
@@ -5,6 +5,7 @@ exports.resolveByokApiKey = resolveByokApiKey;
|
|
|
5
5
|
exports.resolveByokProvider = resolveByokProvider;
|
|
6
6
|
exports.resolveByokModel = resolveByokModel;
|
|
7
7
|
exports.resolveScanAiInference = resolveScanAiInference;
|
|
8
|
+
exports.resolveHostedInferProxyUrl = resolveHostedInferProxyUrl;
|
|
8
9
|
exports.isScanAiDebugEnabled = isScanAiDebugEnabled;
|
|
9
10
|
function resolveWorkspaceApiKey(env) {
|
|
10
11
|
const value = env.DATAPARADE_WORKSPACE_API_KEY?.trim();
|
|
@@ -26,6 +27,10 @@ function resolveScanAiInference(env) {
|
|
|
26
27
|
const value = env.SCAN_AI_INFERENCE?.trim();
|
|
27
28
|
return value || undefined;
|
|
28
29
|
}
|
|
30
|
+
function resolveHostedInferProxyUrl(env) {
|
|
31
|
+
const value = env.SCAN_HOSTED_INFER_PROXY_URL?.trim();
|
|
32
|
+
return value || undefined;
|
|
33
|
+
}
|
|
29
34
|
function parseTruthyEnvFlag(value) {
|
|
30
35
|
if (!value)
|
|
31
36
|
return false;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function resolveSkipAutoUpload(env: NodeJS.ProcessEnv): boolean;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.resolveSkipAutoUpload = resolveSkipAutoUpload;
|
|
4
|
+
const TRUTHY = new Set(["1", "true", "yes", "on"]);
|
|
5
|
+
function resolveSkipAutoUpload(env) {
|
|
6
|
+
const value = env.DATAPARADE_SKIP_AUTO_UPLOAD?.trim().toLowerCase();
|
|
7
|
+
if (!value)
|
|
8
|
+
return false;
|
|
9
|
+
return TRUTHY.has(value);
|
|
10
|
+
}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type { ScanConfiguration } from "../core/types";
|
|
2
2
|
export declare function validateAiInferenceCredentials(config: ScanConfiguration): string[];
|
|
3
|
-
export declare function resolveAiMode(config: ScanConfiguration): "byok" | "platform" | "none";
|
|
3
|
+
export declare function resolveAiMode(config: ScanConfiguration): "byok" | "platform" | "hosted_worker" | "none";
|
|
@@ -6,6 +6,9 @@ function validateAiInferenceCredentials(config) {
|
|
|
6
6
|
const errors = [];
|
|
7
7
|
if (!config.enableAiInference)
|
|
8
8
|
return errors;
|
|
9
|
+
if (config.hostedInferProxyUrl?.trim()) {
|
|
10
|
+
return errors;
|
|
11
|
+
}
|
|
9
12
|
const hasByok = Boolean(config.aiProvider?.trim()) &&
|
|
10
13
|
Boolean(config.aiModel?.trim()) &&
|
|
11
14
|
Boolean(config.aiApiKey?.trim());
|
|
@@ -25,6 +28,8 @@ function validateAiInferenceCredentials(config) {
|
|
|
25
28
|
function resolveAiMode(config) {
|
|
26
29
|
if (!config.enableAiInference)
|
|
27
30
|
return "none";
|
|
31
|
+
if (config.hostedInferProxyUrl?.trim())
|
|
32
|
+
return "hosted_worker";
|
|
28
33
|
if (config.aiApiKey?.trim())
|
|
29
34
|
return "byok";
|
|
30
35
|
if (config.workspaceApiKey?.trim())
|
|
@@ -42,5 +42,10 @@ function buildAgentOrchestratorOptions(config, opts) {
|
|
|
42
42
|
// the helper succeeds (client sees 503 "Service Unavailable").
|
|
43
43
|
base.providerConcurrency = 1;
|
|
44
44
|
}
|
|
45
|
+
if (aiMode === "hosted_worker" && config.hostedInferProxyUrl?.trim()) {
|
|
46
|
+
base.hostedWorkerInferProxyUrl = config.hostedInferProxyUrl.trim();
|
|
47
|
+
base.apiKey = undefined;
|
|
48
|
+
base.providerConcurrency = 1;
|
|
49
|
+
}
|
|
45
50
|
return base;
|
|
46
51
|
}
|
|
@@ -51,10 +51,12 @@ export declare const scanConfigurationSchema: z.ZodObject<{
|
|
|
51
51
|
aiMode: z.ZodOptional<z.ZodEnum<{
|
|
52
52
|
byok: "byok";
|
|
53
53
|
platform: "platform";
|
|
54
|
+
hosted_worker: "hosted_worker";
|
|
54
55
|
none: "none";
|
|
55
56
|
}>>;
|
|
56
57
|
platformApiBaseUrl: z.ZodOptional<z.ZodString>;
|
|
57
58
|
cliQuotaJobId: z.ZodOptional<z.ZodString>;
|
|
59
|
+
hostedInferProxyUrl: z.ZodOptional<z.ZodString>;
|
|
58
60
|
}, z.core.$strip>;
|
|
59
61
|
export declare function parseScanConfiguration(input: unknown): ScanConfiguration;
|
|
60
62
|
export declare function validateScanConfiguration(input: unknown): {
|
|
@@ -45,9 +45,10 @@ exports.scanConfigurationSchema = zod_1.z.object({
|
|
|
45
45
|
aiInferenceScope: zod_1.z.enum(["default", "third_party_only"]).optional(),
|
|
46
46
|
aiVerbose: zod_1.z.boolean().optional(),
|
|
47
47
|
workspaceApiKey: zod_1.z.string().min(1).optional(),
|
|
48
|
-
aiMode: zod_1.z.enum(["byok", "platform", "none"]).optional(),
|
|
48
|
+
aiMode: zod_1.z.enum(["byok", "platform", "hosted_worker", "none"]).optional(),
|
|
49
49
|
platformApiBaseUrl: zod_1.z.string().min(1).optional(),
|
|
50
50
|
cliQuotaJobId: zod_1.z.string().min(1).optional(),
|
|
51
|
+
hostedInferProxyUrl: zod_1.z.string().min(1).optional(),
|
|
51
52
|
});
|
|
52
53
|
function parseScanConfiguration(input) {
|
|
53
54
|
return exports.scanConfigurationSchema.parse(input);
|
|
@@ -49,9 +49,11 @@ export interface ScanConfiguration {
|
|
|
49
49
|
/** DataParade workspace API key (quota + platform LLM proxy). */
|
|
50
50
|
workspaceApiKey?: string;
|
|
51
51
|
/** How LLM inference is billed: byok (direct provider) or platform (API proxy). */
|
|
52
|
-
aiMode?: "byok" | "platform" | "none";
|
|
52
|
+
aiMode?: "byok" | "platform" | "hosted_worker" | "none";
|
|
53
53
|
/** Base URL for platform API (preflight / infer / complete). */
|
|
54
54
|
platformApiBaseUrl?: string;
|
|
55
55
|
/** CLI quota job id from preflight (platform mode). */
|
|
56
56
|
cliQuotaJobId?: string;
|
|
57
|
+
/** Loopback infer URL when hosted scan runs inside VPC scan worker. */
|
|
58
|
+
hostedInferProxyUrl?: string;
|
|
57
59
|
}
|
|
@@ -10,6 +10,8 @@ export interface BuildDataflowWrapperOptions {
|
|
|
10
10
|
* dataflow.json schema version.
|
|
11
11
|
*/
|
|
12
12
|
schemaVersion?: string;
|
|
13
|
+
/** Assessment / project name shown in the dashboard import preview. */
|
|
14
|
+
projectName?: string;
|
|
13
15
|
}
|
|
14
16
|
/**
|
|
15
17
|
* Assemble the top-level `dataflow.json` wrapper object from a scan result and graph.
|
|
@@ -35,6 +37,8 @@ export interface WriteDataflowJsonOptions {
|
|
|
35
37
|
* Optional schema version to pass through to `buildDataflowWrapper`.
|
|
36
38
|
*/
|
|
37
39
|
schemaVersion?: string;
|
|
40
|
+
/** Assessment / project name stored in wrapper metadata for upload and preview. */
|
|
41
|
+
projectName?: string;
|
|
38
42
|
}
|
|
39
43
|
/**
|
|
40
44
|
* Validate and write a `dataflow.json` wrapper to disk.
|
package/dist/src/output/json.js
CHANGED
|
@@ -15,11 +15,13 @@ const dataflow_wrapper_schema_1 = require("../core/schema/dataflow-wrapper.schem
|
|
|
15
15
|
*/
|
|
16
16
|
function buildDataflowWrapper(scanResult, graph, options = {}) {
|
|
17
17
|
const schemaVersion = options.schemaVersion ?? "1.0";
|
|
18
|
+
const projectName = options.projectName?.trim();
|
|
18
19
|
const metadata = {
|
|
19
20
|
componentsCount: scanResult.components.length,
|
|
20
21
|
dataFlowsCount: scanResult.dataFlows.length,
|
|
21
22
|
filesScanned: scanResult.filesScanned,
|
|
22
23
|
scanDurationMs: scanResult.scanDurationMs,
|
|
24
|
+
...(projectName ? { projectName } : {}),
|
|
23
25
|
};
|
|
24
26
|
if (scanResult.aiInferenceSummary) {
|
|
25
27
|
metadata.aiInference =
|
|
@@ -47,8 +49,11 @@ function buildDataflowWrapper(scanResult, graph, options = {}) {
|
|
|
47
49
|
* can emit a non-zero exit code.
|
|
48
50
|
*/
|
|
49
51
|
function writeDataflowJson(options) {
|
|
50
|
-
const { scanResult, graph, outputPath, schemaVersion } = options;
|
|
51
|
-
const wrapper = buildDataflowWrapper(scanResult, graph, {
|
|
52
|
+
const { scanResult, graph, outputPath, schemaVersion, projectName } = options;
|
|
53
|
+
const wrapper = buildDataflowWrapper(scanResult, graph, {
|
|
54
|
+
schemaVersion,
|
|
55
|
+
projectName,
|
|
56
|
+
});
|
|
52
57
|
const validation = (0, dataflow_wrapper_schema_1.validateDataflowJson)(wrapper);
|
|
53
58
|
if (!validation.ok) {
|
|
54
59
|
const messages = validation.errors.join("; ");
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Default DataParade web app (frontend Lambda URL). Used when the CLI prints
|
|
3
|
+
* dashboard preview links. Override with DATAPARADE_APP_URL for local dev.
|
|
4
|
+
*/
|
|
5
|
+
export declare const DEFAULT_DATAPARADE_APP_URL = "https://tse3dlzv73va5vycctveedw27i0xwncj.lambda-url.us-east-1.on.aws";
|
|
6
|
+
/** Resolve web app origin (no trailing slash). */
|
|
7
|
+
export declare function getDataparadeAppBaseUrl(env?: NodeJS.ProcessEnv): string;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DEFAULT_DATAPARADE_APP_URL = void 0;
|
|
4
|
+
exports.getDataparadeAppBaseUrl = getDataparadeAppBaseUrl;
|
|
5
|
+
/**
|
|
6
|
+
* Default DataParade web app (frontend Lambda URL). Used when the CLI prints
|
|
7
|
+
* dashboard preview links. Override with DATAPARADE_APP_URL for local dev.
|
|
8
|
+
*/
|
|
9
|
+
exports.DEFAULT_DATAPARADE_APP_URL = "https://tse3dlzv73va5vycctveedw27i0xwncj.lambda-url.us-east-1.on.aws";
|
|
10
|
+
/** Resolve web app origin (no trailing slash). */
|
|
11
|
+
function getDataparadeAppBaseUrl(env = process.env) {
|
|
12
|
+
const base = env.DATAPARADE_APP_URL?.trim() || exports.DEFAULT_DATAPARADE_APP_URL;
|
|
13
|
+
return base.replace(/\/$/, "");
|
|
14
|
+
}
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import type { CliUploadPreviewInput, CliUploadPreviewResponse } from "./upload.types";
|
|
2
|
+
export type { CliUploadPreviewInput, CliUploadPreviewResponse } from "./upload.types";
|
|
3
|
+
export declare function cliUploadPreview(input: CliUploadPreviewInput): Promise<CliUploadPreviewResponse>;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.cliUploadPreview = cliUploadPreview;
|
|
4
|
+
const dataparade_api_base_url_1 = require("./dataparade-api-base-url");
|
|
5
|
+
function buildHeaders(apiKey) {
|
|
6
|
+
return {
|
|
7
|
+
Authorization: `Bearer ${apiKey}`,
|
|
8
|
+
"Content-Type": "application/json",
|
|
9
|
+
};
|
|
10
|
+
}
|
|
11
|
+
async function cliUploadPreview(input) {
|
|
12
|
+
const res = await fetch(`${(0, dataparade_api_base_url_1.getDataparadeApiBaseUrl)()}/api/scans/cli/upload`, {
|
|
13
|
+
method: "POST",
|
|
14
|
+
headers: buildHeaders(input.apiKey),
|
|
15
|
+
body: JSON.stringify({
|
|
16
|
+
dataflow: input.dataflow,
|
|
17
|
+
projectName: input.projectName,
|
|
18
|
+
scanJobId: input.scanJobId,
|
|
19
|
+
}),
|
|
20
|
+
});
|
|
21
|
+
if (!res.ok) {
|
|
22
|
+
let message = `Upload failed (${res.status})`;
|
|
23
|
+
try {
|
|
24
|
+
const body = (await res.json());
|
|
25
|
+
if (typeof body.message === "string")
|
|
26
|
+
message = body.message;
|
|
27
|
+
else if (Array.isArray(body.message))
|
|
28
|
+
message = body.message.join(", ");
|
|
29
|
+
else if (typeof body.error === "string")
|
|
30
|
+
message = body.error;
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
// ignore
|
|
34
|
+
}
|
|
35
|
+
throw new Error(message);
|
|
36
|
+
}
|
|
37
|
+
return (await res.json());
|
|
38
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function buildImportPreviewUrl(draftId: string): string;
|