@dataparade/cli 0.0.2 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -0
- package/dist/package.json +85 -0
- package/dist/src/ai-enrichment/agent-orchestrator.d.ts +2 -0
- package/dist/src/ai-enrichment/agent-orchestrator.js +8 -5
- package/dist/src/ai-enrichment/providers/hosted-worker-infer-provider.d.ts +10 -0
- package/dist/src/ai-enrichment/providers/hosted-worker-infer-provider.js +47 -0
- package/dist/src/cli.js +85 -1
- package/dist/src/config/env.js +2 -0
- package/dist/src/config/resolve.js +3 -0
- package/dist/src/config/scan-env.d.ts +1 -0
- package/dist/src/config/scan-env.js +5 -0
- package/dist/src/config/types.d.ts +1 -0
- package/dist/src/config/upload-env.d.ts +1 -0
- package/dist/src/config/upload-env.js +10 -0
- package/dist/src/config/validate-scan-ai.d.ts +1 -1
- package/dist/src/config/validate-scan-ai.js +5 -0
- package/dist/src/core/pipeline/ai-orchestrator-options.js +5 -0
- package/dist/src/core/pipeline/graph-mapping.js +5 -0
- package/dist/src/core/pipeline/infer-data-flow-protocol.d.ts +7 -0
- package/dist/src/core/pipeline/infer-data-flow-protocol.js +39 -0
- package/dist/src/core/schema/scan-config.schema.d.ts +2 -0
- package/dist/src/core/schema/scan-config.schema.js +2 -1
- package/dist/src/core/types/config.d.ts +3 -1
- package/dist/src/observability/scan-sentry.d.ts +1 -1
- package/dist/src/platform-api/dataparade-app-base-url.d.ts +7 -0
- package/dist/src/platform-api/dataparade-app-base-url.js +14 -0
- package/dist/src/platform-api/upload-client.d.ts +3 -0
- package/dist/src/platform-api/upload-client.js +38 -0
- package/dist/src/platform-api/upload.types.d.ts +10 -0
- package/dist/src/platform-api/upload.types.js +2 -0
- package/dist/src/upload/build-preview-url.d.ts +1 -0
- package/dist/src/upload/build-preview-url.js +10 -0
- package/dist/src/upload/run-upload.d.ts +2 -0
- package/dist/src/upload/run-upload.js +18 -0
- package/dist/src/upload/types.d.ts +11 -0
- package/dist/src/upload/types.js +2 -0
- package/dist/tests/unit/analyzers/typescript/property-detection.spec.js +22 -0
- package/dist/tests/unit/cli/scan-quota-flow.spec.js +2 -0
- package/dist/tests/unit/config/upload-env.spec.d.ts +1 -0
- package/dist/tests/unit/config/upload-env.spec.js +12 -0
- package/dist/tests/unit/config/validate-scan-ai.spec.js +8 -0
- package/dist/tests/unit/core/graph-mapping.spec.js +52 -0
- package/dist/tests/unit/core/infer-data-flow-protocol.spec.d.ts +1 -0
- package/dist/tests/unit/core/infer-data-flow-protocol.spec.js +37 -0
- package/dist/tests/unit/platform-api/dataparade-app-base-url.spec.d.ts +1 -0
- package/dist/tests/unit/platform-api/dataparade-app-base-url.spec.js +13 -0
- package/dist/tests/unit/platform-api/upload-client.spec.d.ts +1 -0
- package/dist/tests/unit/platform-api/upload-client.spec.js +40 -0
- package/dist/tests/unit/publish-manifest.spec.d.ts +1 -0
- package/dist/tests/unit/publish-manifest.spec.js +18 -0
- package/dist/tests/unit/upload/build-preview-url.spec.d.ts +1 -0
- package/dist/tests/unit/upload/build-preview-url.spec.js +19 -0
- package/package.json +4 -3
- package/patterns/README.md +322 -0
- package/patterns/actor.patterns.yaml +51 -0
- package/patterns/aws-terraform-catalog.snapshot.json +1760 -0
- package/patterns/aws-terraform-service-hints.generated.json +1446 -0
- package/patterns/azure-terraform-catalog.snapshot.json +1195 -0
- package/patterns/azure-terraform-service-hints.generated.json +864 -0
- package/patterns/classifier/actors.classifier.yaml +17 -0
- package/patterns/classifier/components.classifier.yaml +110 -0
- package/patterns/classifier/third-party.classifier.yaml +169 -0
- package/patterns/kubernetes-terraform-catalog.snapshot.json +94 -0
- package/patterns/kubernetes-terraform-service-hints.generated.json +258 -0
- package/patterns/non-pii-signals.rules.yaml +84 -0
- package/patterns/pii-signals.rules.yaml +115 -0
- package/patterns/property.patterns.yaml +431 -0
- package/patterns/provider-topology.rules.yaml +765 -0
- package/patterns/python.md +211 -0
- package/patterns/python.patterns.yaml +160 -0
- package/patterns/terraform.md +37 -0
- package/patterns/terraform.patterns.yaml +495 -0
- package/patterns/third-party.patterns.yaml +167 -0
- package/patterns/typescript.patterns.yaml +139 -0
package/README.md
CHANGED
|
@@ -86,6 +86,30 @@ node dist/bin/cli.js scan path/to/your/project
|
|
|
86
86
|
|
|
87
87
|
---
|
|
88
88
|
|
|
89
|
+
## Upload to dashboard
|
|
90
|
+
|
|
91
|
+
After a scan, the CLI **auto-uploads** `dataflow.json` when `DATAPARADE_WORKSPACE_API_KEY` is set (from **Workspace → Access keys**). This creates an **import preview draft** in the web app — not a finished assessment. Open the printed URL to review and edit before creating the diagram.
|
|
92
|
+
|
|
93
|
+
- Opt out: `--skip-auto-upload` or `DATAPARADE_SKIP_AUTO_UPLOAD=true`
|
|
94
|
+
- Without a workspace key, the scan still succeeds; the CLI suggests running `upload` later
|
|
95
|
+
- Upload alone does **not** consume scan quota (platform AI scans still use preflight/complete as before)
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
# Upload an existing file
|
|
99
|
+
node dist/bin/cli.js upload ./dataflow.json --project-name "My service"
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
Environment:
|
|
103
|
+
|
|
104
|
+
| Variable | Purpose |
|
|
105
|
+
|----------|---------|
|
|
106
|
+
| `DATAPARADE_WORKSPACE_API_KEY` | Auth for upload (same key as platform AI) |
|
|
107
|
+
| `DATAPARADE_API_BASE_URL` | Backend API (default: production AWS) |
|
|
108
|
+
| `DATAPARADE_APP_URL` | Frontend base for preview links (default: production app) |
|
|
109
|
+
| `DATAPARADE_SKIP_AUTO_UPLOAD` | Skip post-scan upload when `true`/`1` |
|
|
110
|
+
|
|
111
|
+
---
|
|
112
|
+
|
|
89
113
|
## Output: `dataflow.json`
|
|
90
114
|
|
|
91
115
|
By default (no `--output` flag), the CLI writes `./dataflow.json` in the current working directory.
|
|
@@ -112,6 +136,7 @@ The CLI emits **all** Engineering, Privacy, and Security property keys (from the
|
|
|
112
136
|
- Database: `connection_encryption`, `backup_frequency`, `audit_logging_enabled`, etc. when patterns match.
|
|
113
137
|
- Auth: `mfa_required`, `authentication_method`, `sso_integration` from auth middleware patterns.
|
|
114
138
|
- Routes: `request_validation`, `api_type`, `https_enforced` for route patterns.
|
|
139
|
+
- Data flow edges (graph mapping): `engineering.protocol` is set to `rest` or `graphql` for `api_call` flows when the endpoint, method, or source code indicates HTTP REST vs GraphQL (see `infer-data-flow-protocol.ts`). Pattern rules in `property.patterns.yaml` also set `api_type` to `graphql` for `/graphql` URLs and paths.
|
|
115
140
|
|
|
116
141
|
**Terraform / IaC (`.tf` / `.tfvars`):**
|
|
117
142
|
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@dataparade/cli",
|
|
3
|
+
"version": "0.0.4",
|
|
4
|
+
"description": "dataPARADE CLI — scan codebases for privacy and security data flows",
|
|
5
|
+
"publishConfig": {
|
|
6
|
+
"access": "public",
|
|
7
|
+
"registry": "https://registry.npmjs.org/"
|
|
8
|
+
},
|
|
9
|
+
"license": "GPL-3.0-or-later",
|
|
10
|
+
"author": "DataParade",
|
|
11
|
+
"homepage": "https://github.com/DataParade-io/dataparade-cli#readme",
|
|
12
|
+
"repository": {
|
|
13
|
+
"type": "git",
|
|
14
|
+
"url": "git+https://github.com/DataParade-io/dataparade-cli.git"
|
|
15
|
+
},
|
|
16
|
+
"bugs": {
|
|
17
|
+
"url": "https://github.com/DataParade-io/dataparade-cli/issues"
|
|
18
|
+
},
|
|
19
|
+
"engines": {
|
|
20
|
+
"node": ">=20"
|
|
21
|
+
},
|
|
22
|
+
"keywords": [
|
|
23
|
+
"dataparade",
|
|
24
|
+
"privacy",
|
|
25
|
+
"data-flow",
|
|
26
|
+
"security",
|
|
27
|
+
"cli",
|
|
28
|
+
"scanner"
|
|
29
|
+
],
|
|
30
|
+
"private": false,
|
|
31
|
+
"main": "dist/src/index.js",
|
|
32
|
+
"types": "dist/src/index.d.ts",
|
|
33
|
+
"exports": {
|
|
34
|
+
".": {
|
|
35
|
+
"types": "./dist/src/index.d.ts",
|
|
36
|
+
"default": "./dist/src/index.js"
|
|
37
|
+
},
|
|
38
|
+
"./ai-providers": {
|
|
39
|
+
"types": "./dist/src/ai-enrichment/providers/index.d.ts",
|
|
40
|
+
"default": "./dist/src/ai-enrichment/providers/index.js"
|
|
41
|
+
}
|
|
42
|
+
},
|
|
43
|
+
"bin": {
|
|
44
|
+
"@dataparade/cli": "dist/bin/cli.js"
|
|
45
|
+
},
|
|
46
|
+
"scripts": {
|
|
47
|
+
"generate:terraform-provider-hints": "node scripts/generate-terraform-provider-hints.mjs",
|
|
48
|
+
"generate:aws-terraform-hints": "node scripts/generate-aws-terraform-hints.mjs",
|
|
49
|
+
"generate:azure-terraform-hints": "node scripts/generate-terraform-provider-hints.mjs --azure-only",
|
|
50
|
+
"generate:kubernetes-terraform-hints": "node scripts/generate-terraform-provider-hints.mjs --kubernetes-only",
|
|
51
|
+
"build": "pnpm exec tsc -p tsconfig.json",
|
|
52
|
+
"prepublishOnly": "pnpm run build && pnpm run test",
|
|
53
|
+
"lint": "eslint \"src/**/*.ts\" \"tests/**/*.ts\"",
|
|
54
|
+
"pretest": "pnpm run build",
|
|
55
|
+
"test": "jest",
|
|
56
|
+
"scan": "node dist/bin/cli.js scan",
|
|
57
|
+
"eval:models": "pnpm exec node dist/src/evals/run-model-matrix.js"
|
|
58
|
+
},
|
|
59
|
+
"dependencies": {
|
|
60
|
+
"@sentry/node": "^10.38.0",
|
|
61
|
+
"commander": "^12.1.0",
|
|
62
|
+
"dotenv": "^16.4.7",
|
|
63
|
+
"langsmith": "^0.3.14",
|
|
64
|
+
"typescript": "^5.7.3",
|
|
65
|
+
"yaml": "^2.8.2",
|
|
66
|
+
"zod": "^4.3.5"
|
|
67
|
+
},
|
|
68
|
+
"devDependencies": {
|
|
69
|
+
"@eslint/js": "^9.18.0",
|
|
70
|
+
"@types/jest": "^29.5.14",
|
|
71
|
+
"@types/node": "^22.0.0",
|
|
72
|
+
"eslint": "^9.18.0",
|
|
73
|
+
"globals": "^15.14.0",
|
|
74
|
+
"jest": "^29.7.0",
|
|
75
|
+
"ts-jest": "^29.2.5",
|
|
76
|
+
"typescript-eslint": "^8.20.0"
|
|
77
|
+
},
|
|
78
|
+
"files": [
|
|
79
|
+
"dist",
|
|
80
|
+
"patterns",
|
|
81
|
+
"LICENSE",
|
|
82
|
+
"README.md",
|
|
83
|
+
"package.json"
|
|
84
|
+
]
|
|
85
|
+
}
|
|
@@ -18,6 +18,8 @@ export interface AgentOrchestratorOptions {
|
|
|
18
18
|
skipStructuralHeuristics?: boolean;
|
|
19
19
|
/** Platform-billed LLM via DataParade API proxy. */
|
|
20
20
|
platformProxy?: PlatformProxyProviderConfig;
|
|
21
|
+
/** Hosted scan in VPC worker: loopback proxy → scan-cli-ai-helper Lambda. */
|
|
22
|
+
hostedWorkerInferProxyUrl?: string;
|
|
21
23
|
}
|
|
22
24
|
export declare function generateAgenticProposals(input: RunInferencePipelineInput, plan: InferencePlannerResult, candidates: AiInferenceCandidate[], options: AgentOrchestratorOptions): Promise<{
|
|
23
25
|
proposals: Array<{
|
|
@@ -10,6 +10,7 @@ const third_party_evidence_1 = require("./third-party-evidence");
|
|
|
10
10
|
const provider_contract_1 = require("./providers/provider-contract");
|
|
11
11
|
const scan_paths_1 = require("./scan-paths");
|
|
12
12
|
const platform_proxy_provider_1 = require("./providers/platform-proxy-provider");
|
|
13
|
+
const hosted_worker_infer_provider_1 = require("./providers/hosted-worker-infer-provider");
|
|
13
14
|
const PROPERTY_AGENT_SLICE_SIZE = 12;
|
|
14
15
|
const TP_AGENT_MAX_ROUNDS = 3;
|
|
15
16
|
const TP_AGENT_MAX_FILES = 8;
|
|
@@ -156,11 +157,13 @@ async function generateAgenticProposals(input, plan, candidates, options) {
|
|
|
156
157
|
const skipStructural = options.skipStructuralHeuristics === true;
|
|
157
158
|
const provider = options.platformProxy
|
|
158
159
|
? new platform_proxy_provider_1.PlatformProxyProvider(options.platformProxy)
|
|
159
|
-
: (
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
160
|
+
: options.hostedWorkerInferProxyUrl?.trim()
|
|
161
|
+
? new hosted_worker_infer_provider_1.HostedWorkerInferProvider(options.hostedWorkerInferProxyUrl.trim())
|
|
162
|
+
: (0, providers_1.createAiProvider)(options.provider, {
|
|
163
|
+
endpoint: options.endpoint,
|
|
164
|
+
model: options.model,
|
|
165
|
+
apiKey: options.apiKey,
|
|
166
|
+
});
|
|
164
167
|
const generated = [];
|
|
165
168
|
const agenticTrace = [];
|
|
166
169
|
const aggregateUsage = {
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { AiProvider, AiProviderRequest, AiProviderResult } from "./types";
|
|
2
|
+
/**
|
|
3
|
+
* Sync infer via the scan worker's loopback proxy (VPC → scan-cli-ai-helper Lambda).
|
|
4
|
+
*/
|
|
5
|
+
export declare class HostedWorkerInferProvider implements AiProvider {
|
|
6
|
+
private readonly inferUrl;
|
|
7
|
+
readonly id: "openai";
|
|
8
|
+
constructor(inferUrl: string);
|
|
9
|
+
infer(request: AiProviderRequest): Promise<AiProviderResult>;
|
|
10
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.HostedWorkerInferProvider = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* Sync infer via the scan worker's loopback proxy (VPC → scan-cli-ai-helper Lambda).
|
|
6
|
+
*/
|
|
7
|
+
class HostedWorkerInferProvider {
|
|
8
|
+
constructor(inferUrl) {
|
|
9
|
+
this.inferUrl = inferUrl;
|
|
10
|
+
this.id = "openai";
|
|
11
|
+
}
|
|
12
|
+
async infer(request) {
|
|
13
|
+
const res = await fetch(this.inferUrl, {
|
|
14
|
+
method: "POST",
|
|
15
|
+
headers: { "Content-Type": "application/json" },
|
|
16
|
+
body: JSON.stringify({
|
|
17
|
+
prompt: request.prompt,
|
|
18
|
+
maxTokens: request.maxTokens,
|
|
19
|
+
temperature: request.temperature,
|
|
20
|
+
}),
|
|
21
|
+
});
|
|
22
|
+
let body;
|
|
23
|
+
try {
|
|
24
|
+
body = (await res.json());
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
throw new Error(`Hosted scan AI infer failed (${res.status})`);
|
|
28
|
+
}
|
|
29
|
+
if (!res.ok) {
|
|
30
|
+
throw new Error(body.error?.trim() ||
|
|
31
|
+
`Hosted scan AI infer failed (${res.status})`);
|
|
32
|
+
}
|
|
33
|
+
const proposals = (Array.isArray(body.proposals) ? body.proposals : []);
|
|
34
|
+
const usage = body.usage;
|
|
35
|
+
return {
|
|
36
|
+
proposals,
|
|
37
|
+
usage: usage && typeof usage.totalTokens === "number"
|
|
38
|
+
? {
|
|
39
|
+
inputTokens: Math.max(0, usage.inputTokens ?? 0),
|
|
40
|
+
outputTokens: Math.max(0, usage.outputTokens ?? 0),
|
|
41
|
+
totalTokens: Math.max(0, usage.totalTokens),
|
|
42
|
+
}
|
|
43
|
+
: undefined,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
exports.HostedWorkerInferProvider = HostedWorkerInferProvider;
|
package/dist/src/cli.js
CHANGED
|
@@ -40,7 +40,9 @@ exports.run = run;
|
|
|
40
40
|
require("./config/load-cli-env");
|
|
41
41
|
const path_1 = __importDefault(require("path"));
|
|
42
42
|
const commander_1 = require("commander");
|
|
43
|
+
const package_json_1 = __importDefault(require("../package.json"));
|
|
43
44
|
const json_1 = require("./output/json");
|
|
45
|
+
const upload_env_1 = require("./config/upload-env");
|
|
44
46
|
const types_1 = require("./ai-enrichment/types");
|
|
45
47
|
const inference_scope_1 = require("./config/inference-scope");
|
|
46
48
|
const redact_1 = require("./config/redact");
|
|
@@ -112,7 +114,7 @@ function createProgram() {
|
|
|
112
114
|
program
|
|
113
115
|
.name("dataparade")
|
|
114
116
|
.description("DataParade CLI - scan codebases for data flow components")
|
|
115
|
-
.version(
|
|
117
|
+
.version(package_json_1.default.version);
|
|
116
118
|
program
|
|
117
119
|
.command("scan <path>")
|
|
118
120
|
.description("Scan a directory (or a single supported source file) for data flow components")
|
|
@@ -143,6 +145,7 @@ function createProgram() {
|
|
|
143
145
|
.option("--api-key <key>", "(deprecated) alias for --workspace-api-key")
|
|
144
146
|
.option("--byok-provider <provider>", `BYOK LLM provider when using your own API key: ${types_1.AI_PROVIDER_IDS.join("|")}`)
|
|
145
147
|
.option("--byok-model <model>", "BYOK model name (env: SCAN_BYOK_MODEL)")
|
|
148
|
+
.option("--skip-auto-upload", "Do not upload dataflow.json to the dashboard after scan (env: DATAPARADE_SKIP_AUTO_UPLOAD)")
|
|
146
149
|
.action(async (path, options) => {
|
|
147
150
|
let cliQuotaJobId;
|
|
148
151
|
let platformQuotaApiKey;
|
|
@@ -378,6 +381,35 @@ function createProgram() {
|
|
|
378
381
|
// tests can rely on it.
|
|
379
382
|
// eslint-disable-next-line no-console
|
|
380
383
|
console.log(`[scan] dataflow.json written to ${dataflowOutputPath}`);
|
|
384
|
+
const skipAutoUpload = Boolean(options.skipAutoUpload) ||
|
|
385
|
+
(0, upload_env_1.resolveSkipAutoUpload)(process.env);
|
|
386
|
+
if (!skipAutoUpload) {
|
|
387
|
+
const uploadApiKey = workspaceApiKey;
|
|
388
|
+
if (!uploadApiKey) {
|
|
389
|
+
// eslint-disable-next-line no-console
|
|
390
|
+
console.log(`[scan] Auto-upload skipped (no workspace API key). Run: dataparade upload ${dataflowOutputPath}`);
|
|
391
|
+
}
|
|
392
|
+
else {
|
|
393
|
+
try {
|
|
394
|
+
const { runDataflowUpload } = await Promise.resolve().then(() => __importStar(require("./upload/run-upload")));
|
|
395
|
+
const dataflowWrapper = (0, json_1.buildDataflowWrapper)(scanResult, diagramGraph);
|
|
396
|
+
await runDataflowUpload({
|
|
397
|
+
apiKey: uploadApiKey,
|
|
398
|
+
dataflow: dataflowWrapper,
|
|
399
|
+
projectName: config.projectName,
|
|
400
|
+
scanJobId: cliQuotaJobId,
|
|
401
|
+
logPrefix: "[scan]",
|
|
402
|
+
});
|
|
403
|
+
}
|
|
404
|
+
catch (uploadError) {
|
|
405
|
+
const uploadMessage = uploadError instanceof Error
|
|
406
|
+
? uploadError.message
|
|
407
|
+
: "Unknown upload error.";
|
|
408
|
+
// eslint-disable-next-line no-console
|
|
409
|
+
console.error(`[scan] Auto-upload failed: ${uploadMessage}`);
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
}
|
|
381
413
|
}
|
|
382
414
|
catch (dataflowError) {
|
|
383
415
|
await (0, scan_sentry_1.reportScanCliError)({
|
|
@@ -453,6 +485,58 @@ function createProgram() {
|
|
|
453
485
|
}
|
|
454
486
|
}
|
|
455
487
|
});
|
|
488
|
+
program
|
|
489
|
+
.command("upload <file>")
|
|
490
|
+
.description("Upload a dataflow.json file to the dashboard as an import preview")
|
|
491
|
+
.option("--project-name <name>", "Assessment name shown in the preview (default: from file metadata if present)")
|
|
492
|
+
.option("--workspace-api-key <key>", "DataParade workspace API key (env: DATAPARADE_WORKSPACE_API_KEY)")
|
|
493
|
+
.option("--api-key <key>", "(deprecated) alias for --workspace-api-key")
|
|
494
|
+
.action(async (file, options) => {
|
|
495
|
+
const apiKey = options.workspaceApiKey?.trim() ||
|
|
496
|
+
options.apiKey?.trim() ||
|
|
497
|
+
(0, scan_env_1.resolveWorkspaceApiKey)(process.env);
|
|
498
|
+
if (!apiKey) {
|
|
499
|
+
// eslint-disable-next-line no-console
|
|
500
|
+
console.error("[upload] error: workspace API key is required (DATAPARADE_WORKSPACE_API_KEY or --workspace-api-key)");
|
|
501
|
+
process.exitCode = 1;
|
|
502
|
+
return;
|
|
503
|
+
}
|
|
504
|
+
const filePath = path_1.default.resolve(process.cwd(), file || "dataflow.json");
|
|
505
|
+
let dataflow;
|
|
506
|
+
try {
|
|
507
|
+
const { readFileSync } = await Promise.resolve().then(() => __importStar(require("fs")));
|
|
508
|
+
dataflow = JSON.parse(readFileSync(filePath, "utf8"));
|
|
509
|
+
}
|
|
510
|
+
catch (err) {
|
|
511
|
+
const message = err instanceof Error ? err.message : "Failed to read dataflow.json";
|
|
512
|
+
// eslint-disable-next-line no-console
|
|
513
|
+
console.error(`[upload] error: ${message}`);
|
|
514
|
+
process.exitCode = 1;
|
|
515
|
+
return;
|
|
516
|
+
}
|
|
517
|
+
let projectName = options.projectName?.trim();
|
|
518
|
+
if (!projectName && dataflow && typeof dataflow === "object") {
|
|
519
|
+
const meta = dataflow
|
|
520
|
+
.metadata;
|
|
521
|
+
if (typeof meta?.projectName === "string" && meta.projectName.trim()) {
|
|
522
|
+
projectName = meta.projectName.trim();
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
try {
|
|
526
|
+
const { runDataflowUpload } = await Promise.resolve().then(() => __importStar(require("./upload/run-upload")));
|
|
527
|
+
await runDataflowUpload({
|
|
528
|
+
apiKey,
|
|
529
|
+
dataflow,
|
|
530
|
+
projectName,
|
|
531
|
+
});
|
|
532
|
+
}
|
|
533
|
+
catch (err) {
|
|
534
|
+
const message = err instanceof Error ? err.message : "Upload failed.";
|
|
535
|
+
// eslint-disable-next-line no-console
|
|
536
|
+
console.error(`[upload] error: ${message}`);
|
|
537
|
+
process.exitCode = 1;
|
|
538
|
+
}
|
|
539
|
+
});
|
|
456
540
|
program
|
|
457
541
|
.command("config")
|
|
458
542
|
.description("View the effective configuration for the current project")
|
package/dist/src/config/env.js
CHANGED
|
@@ -50,6 +50,7 @@ function loadCliConfigEnv(env) {
|
|
|
50
50
|
const aiToolLoopMaxSearches = parseNumber(env.SCAN_AI_TOOL_LOOP_MAX_SEARCHES);
|
|
51
51
|
const aiThirdPartyDataFlowEnabled = parseBoolean(env.SCAN_AI_THIRD_PARTY_DATA_FLOW);
|
|
52
52
|
const workspaceApiKey = (0, scan_env_1.resolveWorkspaceApiKey)(env);
|
|
53
|
+
const hostedInferProxyUrl = (0, scan_env_1.resolveHostedInferProxyUrl)(env);
|
|
53
54
|
return {
|
|
54
55
|
excludePaths,
|
|
55
56
|
minimumConfidence,
|
|
@@ -71,5 +72,6 @@ function loadCliConfigEnv(env) {
|
|
|
71
72
|
aiToolLoopMaxSearches,
|
|
72
73
|
aiThirdPartyDataFlowEnabled,
|
|
73
74
|
workspaceApiKey: workspaceApiKey || undefined,
|
|
75
|
+
hostedInferProxyUrl: hostedInferProxyUrl || undefined,
|
|
74
76
|
};
|
|
75
77
|
}
|
|
@@ -283,6 +283,9 @@ function resolveScanConfiguration(options) {
|
|
|
283
283
|
if (envConfig.workspaceApiKey) {
|
|
284
284
|
overrides.workspaceApiKey = envConfig.workspaceApiKey;
|
|
285
285
|
}
|
|
286
|
+
if (envConfig.hostedInferProxyUrl) {
|
|
287
|
+
overrides.hostedInferProxyUrl = envConfig.hostedInferProxyUrl;
|
|
288
|
+
}
|
|
286
289
|
if (envConfig.aiModel) {
|
|
287
290
|
overrides.aiModel = envConfig.aiModel;
|
|
288
291
|
}
|
|
@@ -3,5 +3,6 @@ export declare function resolveByokApiKey(env: NodeJS.ProcessEnv): string | unde
|
|
|
3
3
|
export declare function resolveByokProvider(env: NodeJS.ProcessEnv): string | undefined;
|
|
4
4
|
export declare function resolveByokModel(env: NodeJS.ProcessEnv): string | undefined;
|
|
5
5
|
export declare function resolveScanAiInference(env: NodeJS.ProcessEnv): string | undefined;
|
|
6
|
+
export declare function resolveHostedInferProxyUrl(env: NodeJS.ProcessEnv): string | undefined;
|
|
6
7
|
/** Extra provider HTTP / normalization logging (all providers, not OpenAI-only). */
|
|
7
8
|
export declare function isScanAiDebugEnabled(env?: NodeJS.ProcessEnv): boolean;
|
|
@@ -5,6 +5,7 @@ exports.resolveByokApiKey = resolveByokApiKey;
|
|
|
5
5
|
exports.resolveByokProvider = resolveByokProvider;
|
|
6
6
|
exports.resolveByokModel = resolveByokModel;
|
|
7
7
|
exports.resolveScanAiInference = resolveScanAiInference;
|
|
8
|
+
exports.resolveHostedInferProxyUrl = resolveHostedInferProxyUrl;
|
|
8
9
|
exports.isScanAiDebugEnabled = isScanAiDebugEnabled;
|
|
9
10
|
function resolveWorkspaceApiKey(env) {
|
|
10
11
|
const value = env.DATAPARADE_WORKSPACE_API_KEY?.trim();
|
|
@@ -26,6 +27,10 @@ function resolveScanAiInference(env) {
|
|
|
26
27
|
const value = env.SCAN_AI_INFERENCE?.trim();
|
|
27
28
|
return value || undefined;
|
|
28
29
|
}
|
|
30
|
+
function resolveHostedInferProxyUrl(env) {
|
|
31
|
+
const value = env.SCAN_HOSTED_INFER_PROXY_URL?.trim();
|
|
32
|
+
return value || undefined;
|
|
33
|
+
}
|
|
29
34
|
function parseTruthyEnvFlag(value) {
|
|
30
35
|
if (!value)
|
|
31
36
|
return false;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function resolveSkipAutoUpload(env: NodeJS.ProcessEnv): boolean;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.resolveSkipAutoUpload = resolveSkipAutoUpload;
|
|
4
|
+
const TRUTHY = new Set(["1", "true", "yes", "on"]);
|
|
5
|
+
function resolveSkipAutoUpload(env) {
|
|
6
|
+
const value = env.DATAPARADE_SKIP_AUTO_UPLOAD?.trim().toLowerCase();
|
|
7
|
+
if (!value)
|
|
8
|
+
return false;
|
|
9
|
+
return TRUTHY.has(value);
|
|
10
|
+
}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type { ScanConfiguration } from "../core/types";
|
|
2
2
|
export declare function validateAiInferenceCredentials(config: ScanConfiguration): string[];
|
|
3
|
-
export declare function resolveAiMode(config: ScanConfiguration): "byok" | "platform" | "none";
|
|
3
|
+
export declare function resolveAiMode(config: ScanConfiguration): "byok" | "platform" | "hosted_worker" | "none";
|
|
@@ -6,6 +6,9 @@ function validateAiInferenceCredentials(config) {
|
|
|
6
6
|
const errors = [];
|
|
7
7
|
if (!config.enableAiInference)
|
|
8
8
|
return errors;
|
|
9
|
+
if (config.hostedInferProxyUrl?.trim()) {
|
|
10
|
+
return errors;
|
|
11
|
+
}
|
|
9
12
|
const hasByok = Boolean(config.aiProvider?.trim()) &&
|
|
10
13
|
Boolean(config.aiModel?.trim()) &&
|
|
11
14
|
Boolean(config.aiApiKey?.trim());
|
|
@@ -25,6 +28,8 @@ function validateAiInferenceCredentials(config) {
|
|
|
25
28
|
function resolveAiMode(config) {
|
|
26
29
|
if (!config.enableAiInference)
|
|
27
30
|
return "none";
|
|
31
|
+
if (config.hostedInferProxyUrl?.trim())
|
|
32
|
+
return "hosted_worker";
|
|
28
33
|
if (config.aiApiKey?.trim())
|
|
29
34
|
return "byok";
|
|
30
35
|
if (config.workspaceApiKey?.trim())
|
|
@@ -42,5 +42,10 @@ function buildAgentOrchestratorOptions(config, opts) {
|
|
|
42
42
|
// the helper succeeds (client sees 503 "Service Unavailable").
|
|
43
43
|
base.providerConcurrency = 1;
|
|
44
44
|
}
|
|
45
|
+
if (aiMode === "hosted_worker" && config.hostedInferProxyUrl?.trim()) {
|
|
46
|
+
base.hostedWorkerInferProxyUrl = config.hostedInferProxyUrl.trim();
|
|
47
|
+
base.apiKey = undefined;
|
|
48
|
+
base.providerConcurrency = 1;
|
|
49
|
+
}
|
|
45
50
|
return base;
|
|
46
51
|
}
|
|
@@ -10,6 +10,7 @@ const minimal_terraform_layout_1 = require("./diagram-layout/minimal-terraform-l
|
|
|
10
10
|
const section_helpers_1 = require("./diagram-layout/section-helpers");
|
|
11
11
|
const terraform_lane_layout_1 = require("./diagram-layout/terraform-lane-layout");
|
|
12
12
|
const terraform_minimal_services_1 = require("./terraform-minimal-services");
|
|
13
|
+
const infer_data_flow_protocol_1 = require("./infer-data-flow-protocol");
|
|
13
14
|
function stripCodeFromSourceLocation(loc) {
|
|
14
15
|
if (!loc)
|
|
15
16
|
return undefined;
|
|
@@ -157,6 +158,10 @@ function mapDataFlowToEdge(flow, componentsById) {
|
|
|
157
158
|
if (flow.actions && flow.actions.length > 0) {
|
|
158
159
|
engineering.actions = flow.actions;
|
|
159
160
|
}
|
|
161
|
+
const protocol = (0, infer_data_flow_protocol_1.inferDataFlowProtocol)(flow);
|
|
162
|
+
if (protocol) {
|
|
163
|
+
engineering.protocol = protocol;
|
|
164
|
+
}
|
|
160
165
|
const privacy = {
|
|
161
166
|
dataCategories: flow.dataCategories ?? [],
|
|
162
167
|
dataSubjectCategories: flow.dataSubjectCategories,
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { DetectedDataFlow } from "../types/data-flow";
|
|
2
|
+
export type InferredDataFlowProtocol = "rest" | "graphql";
|
|
3
|
+
/**
|
|
4
|
+
* Infer engineering.protocol for a scanned data flow (rest | graphql).
|
|
5
|
+
* Returns undefined when the flow type is not HTTP/API-shaped or there is no signal.
|
|
6
|
+
*/
|
|
7
|
+
export declare function inferDataFlowProtocol(flow: DetectedDataFlow): InferredDataFlowProtocol | undefined;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.inferDataFlowProtocol = inferDataFlowProtocol;
|
|
4
|
+
const GRAPHQL_SIGNAL = /\bgraphql\b|\/graphql\b/i;
|
|
5
|
+
function textSuggestsGraphql(text) {
|
|
6
|
+
return GRAPHQL_SIGNAL.test(text);
|
|
7
|
+
}
|
|
8
|
+
function flowCodeSuggestsGraphql(flow) {
|
|
9
|
+
const locations = [
|
|
10
|
+
flow.sourceLocation,
|
|
11
|
+
...(flow.sourceLocations ?? []),
|
|
12
|
+
].filter((loc) => loc != null);
|
|
13
|
+
for (const loc of locations) {
|
|
14
|
+
if (typeof loc.code === "string" && textSuggestsGraphql(loc.code)) {
|
|
15
|
+
return true;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
return false;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Infer engineering.protocol for a scanned data flow (rest | graphql).
|
|
22
|
+
* Returns undefined when the flow type is not HTTP/API-shaped or there is no signal.
|
|
23
|
+
*/
|
|
24
|
+
function inferDataFlowProtocol(flow) {
|
|
25
|
+
const endpoint = flow.endpoint?.trim() ?? "";
|
|
26
|
+
if (endpoint && textSuggestsGraphql(endpoint)) {
|
|
27
|
+
return "graphql";
|
|
28
|
+
}
|
|
29
|
+
if (flowCodeSuggestsGraphql(flow)) {
|
|
30
|
+
return "graphql";
|
|
31
|
+
}
|
|
32
|
+
if (flow.type !== "api_call") {
|
|
33
|
+
return undefined;
|
|
34
|
+
}
|
|
35
|
+
if (endpoint || flow.method?.trim()) {
|
|
36
|
+
return "rest";
|
|
37
|
+
}
|
|
38
|
+
return undefined;
|
|
39
|
+
}
|
|
@@ -51,10 +51,12 @@ export declare const scanConfigurationSchema: z.ZodObject<{
|
|
|
51
51
|
aiMode: z.ZodOptional<z.ZodEnum<{
|
|
52
52
|
byok: "byok";
|
|
53
53
|
platform: "platform";
|
|
54
|
+
hosted_worker: "hosted_worker";
|
|
54
55
|
none: "none";
|
|
55
56
|
}>>;
|
|
56
57
|
platformApiBaseUrl: z.ZodOptional<z.ZodString>;
|
|
57
58
|
cliQuotaJobId: z.ZodOptional<z.ZodString>;
|
|
59
|
+
hostedInferProxyUrl: z.ZodOptional<z.ZodString>;
|
|
58
60
|
}, z.core.$strip>;
|
|
59
61
|
export declare function parseScanConfiguration(input: unknown): ScanConfiguration;
|
|
60
62
|
export declare function validateScanConfiguration(input: unknown): {
|
|
@@ -45,9 +45,10 @@ exports.scanConfigurationSchema = zod_1.z.object({
|
|
|
45
45
|
aiInferenceScope: zod_1.z.enum(["default", "third_party_only"]).optional(),
|
|
46
46
|
aiVerbose: zod_1.z.boolean().optional(),
|
|
47
47
|
workspaceApiKey: zod_1.z.string().min(1).optional(),
|
|
48
|
-
aiMode: zod_1.z.enum(["byok", "platform", "none"]).optional(),
|
|
48
|
+
aiMode: zod_1.z.enum(["byok", "platform", "hosted_worker", "none"]).optional(),
|
|
49
49
|
platformApiBaseUrl: zod_1.z.string().min(1).optional(),
|
|
50
50
|
cliQuotaJobId: zod_1.z.string().min(1).optional(),
|
|
51
|
+
hostedInferProxyUrl: zod_1.z.string().min(1).optional(),
|
|
51
52
|
});
|
|
52
53
|
function parseScanConfiguration(input) {
|
|
53
54
|
return exports.scanConfigurationSchema.parse(input);
|
|
@@ -49,9 +49,11 @@ export interface ScanConfiguration {
|
|
|
49
49
|
/** DataParade workspace API key (quota + platform LLM proxy). */
|
|
50
50
|
workspaceApiKey?: string;
|
|
51
51
|
/** How LLM inference is billed: byok (direct provider) or platform (API proxy). */
|
|
52
|
-
aiMode?: "byok" | "platform" | "none";
|
|
52
|
+
aiMode?: "byok" | "platform" | "hosted_worker" | "none";
|
|
53
53
|
/** Base URL for platform API (preflight / infer / complete). */
|
|
54
54
|
platformApiBaseUrl?: string;
|
|
55
55
|
/** CLI quota job id from preflight (platform mode). */
|
|
56
56
|
cliQuotaJobId?: string;
|
|
57
|
+
/** Loopback infer URL when hosted scan runs inside VPC scan worker. */
|
|
58
|
+
hostedInferProxyUrl?: string;
|
|
57
59
|
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Default DataParade web app (frontend Lambda URL). Used when the CLI prints
|
|
3
|
+
* dashboard preview links. Override with DATAPARADE_APP_URL for local dev.
|
|
4
|
+
*/
|
|
5
|
+
export declare const DEFAULT_DATAPARADE_APP_URL = "https://tse3dlzv73va5vycctveedw27i0xwncj.lambda-url.us-east-1.on.aws";
|
|
6
|
+
/** Resolve web app origin (no trailing slash). */
|
|
7
|
+
export declare function getDataparadeAppBaseUrl(env?: NodeJS.ProcessEnv): string;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DEFAULT_DATAPARADE_APP_URL = void 0;
|
|
4
|
+
exports.getDataparadeAppBaseUrl = getDataparadeAppBaseUrl;
|
|
5
|
+
/**
|
|
6
|
+
* Default DataParade web app (frontend Lambda URL). Used when the CLI prints
|
|
7
|
+
* dashboard preview links. Override with DATAPARADE_APP_URL for local dev.
|
|
8
|
+
*/
|
|
9
|
+
exports.DEFAULT_DATAPARADE_APP_URL = "https://tse3dlzv73va5vycctveedw27i0xwncj.lambda-url.us-east-1.on.aws";
|
|
10
|
+
/** Resolve web app origin (no trailing slash). */
|
|
11
|
+
function getDataparadeAppBaseUrl(env = process.env) {
|
|
12
|
+
const base = env.DATAPARADE_APP_URL?.trim() || exports.DEFAULT_DATAPARADE_APP_URL;
|
|
13
|
+
return base.replace(/\/$/, "");
|
|
14
|
+
}
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import type { CliUploadPreviewInput, CliUploadPreviewResponse } from "./upload.types";
|
|
2
|
+
export type { CliUploadPreviewInput, CliUploadPreviewResponse } from "./upload.types";
|
|
3
|
+
export declare function cliUploadPreview(input: CliUploadPreviewInput): Promise<CliUploadPreviewResponse>;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.cliUploadPreview = cliUploadPreview;
|
|
4
|
+
const dataparade_api_base_url_1 = require("./dataparade-api-base-url");
|
|
5
|
+
function buildHeaders(apiKey) {
|
|
6
|
+
return {
|
|
7
|
+
Authorization: `Bearer ${apiKey}`,
|
|
8
|
+
"Content-Type": "application/json",
|
|
9
|
+
};
|
|
10
|
+
}
|
|
11
|
+
async function cliUploadPreview(input) {
|
|
12
|
+
const res = await fetch(`${(0, dataparade_api_base_url_1.getDataparadeApiBaseUrl)()}/api/scans/cli/upload`, {
|
|
13
|
+
method: "POST",
|
|
14
|
+
headers: buildHeaders(input.apiKey),
|
|
15
|
+
body: JSON.stringify({
|
|
16
|
+
dataflow: input.dataflow,
|
|
17
|
+
projectName: input.projectName,
|
|
18
|
+
scanJobId: input.scanJobId,
|
|
19
|
+
}),
|
|
20
|
+
});
|
|
21
|
+
if (!res.ok) {
|
|
22
|
+
let message = `Upload failed (${res.status})`;
|
|
23
|
+
try {
|
|
24
|
+
const body = (await res.json());
|
|
25
|
+
if (typeof body.message === "string")
|
|
26
|
+
message = body.message;
|
|
27
|
+
else if (Array.isArray(body.message))
|
|
28
|
+
message = body.message.join(", ");
|
|
29
|
+
else if (typeof body.error === "string")
|
|
30
|
+
message = body.error;
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
// ignore
|
|
34
|
+
}
|
|
35
|
+
throw new Error(message);
|
|
36
|
+
}
|
|
37
|
+
return (await res.json());
|
|
38
|
+
}
|