@dataparade/cli 0.0.6 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/package.json +2 -2
- package/dist/src/ai-enrichment/providers/families/chat-completions-family.js +4 -1
- package/dist/src/ai-enrichment/providers/families/generate-content-family.js +3 -1
- package/dist/src/ai-enrichment/providers/families/messages-family.js +1 -1
- package/dist/src/ai-enrichment/providers/families/ollama-generate-family.js +1 -1
- package/dist/src/ai-enrichment/providers/types.d.ts +6 -0
- package/dist/src/cli.js +85 -41
- package/dist/src/config/env.js +7 -2
- package/dist/src/config/load-cli-env.d.ts +2 -0
- package/dist/src/config/load-cli-env.js +11 -2
- package/dist/src/config/redact.js +3 -0
- package/dist/src/config/validate-scan-ai.js +13 -7
- package/dist/src/core/pipeline/ai-orchestrator-options.js +8 -4
- package/dist/src/core/pipeline/orchestrator.js +1 -1
- package/dist/src/core/schema/scan-config.schema.d.ts +1 -0
- package/dist/src/core/schema/scan-config.schema.js +1 -0
- package/dist/src/core/types/config.d.ts +5 -0
- package/dist/src/platform-api/anonymous-ai-session-client.d.ts +14 -0
- package/dist/src/platform-api/anonymous-ai-session-client.js +40 -0
- package/dist/src/platform-api/read-cli-api-error-message.d.ts +18 -0
- package/dist/src/platform-api/read-cli-api-error-message.js +31 -0
- package/dist/src/platform-api/scan-quota-client.d.ts +5 -0
- package/dist/src/platform-api/scan-quota-client.js +24 -14
- package/dist/src/platform-api/upload-client.d.ts +10 -3
- package/dist/src/platform-api/upload-client.js +35 -16
- package/dist/src/platform-api/upload.types.d.ts +10 -5
- package/dist/src/upload/build-preview-url.d.ts +1 -0
- package/dist/src/upload/build-preview-url.js +6 -0
- package/dist/src/upload/run-upload.js +20 -4
- package/dist/src/upload/types.d.ts +2 -2
- package/dist/tests/cli/cli.spec.js +17 -3
- package/dist/tests/unit/ai-enrichment/provider-prompt-backend-contract.spec.d.ts +1 -0
- package/dist/tests/unit/ai-enrichment/provider-prompt-backend-contract.spec.js +134 -0
- package/dist/tests/unit/ai-enrichment/providers/system-prompt-override.spec.d.ts +1 -0
- package/dist/tests/unit/ai-enrichment/providers/system-prompt-override.spec.js +91 -0
- package/dist/tests/unit/cli/scan-command-ai-flags.spec.js +19 -0
- package/dist/tests/unit/cli/scan-command-diagram-graph.spec.js +10 -1
- package/dist/tests/unit/cli/scan-command-e2e-monorepo-app-plus-terraform.spec.js +12 -8
- package/dist/tests/unit/cli/scan-command-e2e-monorepo-front-back-sections.spec.js +10 -1
- package/dist/tests/unit/cli/scan-command-e2e-python-basic.spec.js +10 -1
- package/dist/tests/unit/cli/scan-command-e2e-terraform-basic.spec.js +12 -8
- package/dist/tests/unit/cli/scan-command-e2e-ts-sample.spec.js +10 -1
- package/dist/tests/unit/cli/scan-command-e2e-ts-sentry-route-env.spec.js +1 -1
- package/dist/tests/unit/cli/scan-command-exit-code.spec.js +23 -0
- package/dist/tests/unit/cli/scan-quota-flow.spec.js +44 -1
- package/dist/tests/unit/config/load-cli-env.spec.d.ts +1 -0
- package/dist/tests/unit/config/load-cli-env.spec.js +52 -0
- package/dist/tests/unit/config/redact.spec.js +1 -1
- package/dist/tests/unit/config/resolve-config.spec.js +1 -1
- package/dist/tests/unit/config/validate-scan-ai.spec.js +25 -0
- package/dist/tests/unit/core/ai-orchestrator-options-anon.spec.d.ts +1 -0
- package/dist/tests/unit/core/ai-orchestrator-options-anon.spec.js +29 -0
- package/dist/tests/unit/core/dependency-manifest-performance-budgets.spec.js +1 -1
- package/dist/tests/unit/core/graph-mapping.spec.js +1 -1
- package/dist/tests/unit/core/invariant-tests-dedupe-determinism.spec.js +1 -1
- package/dist/tests/unit/core/no-source-code-leakage.spec.js +1 -1
- package/dist/tests/unit/core/orchestrator.spec.js +12 -12
- package/dist/tests/unit/core/output-json.spec.js +2 -2
- package/dist/tests/unit/core/parsing-error-handling-nonfatal.spec.js +1 -1
- package/dist/tests/unit/core/scan-result-language-stats.spec.js +1 -1
- package/dist/tests/unit/core/structural-scan.spec.js +1 -1
- package/dist/tests/unit/core/terraform-json-overlay.spec.js +1 -1
- package/dist/tests/unit/platform-api/anonymous-ai-session-client.spec.d.ts +1 -0
- package/dist/tests/unit/platform-api/anonymous-ai-session-client.spec.js +35 -0
- package/dist/tests/unit/platform-api/read-cli-api-error-message.spec.d.ts +1 -0
- package/dist/tests/unit/platform-api/read-cli-api-error-message.spec.js +38 -0
- package/dist/tests/unit/upload/build-preview-url.spec.js +16 -0
- package/dist/tests/unit/upload/run-upload.spec.d.ts +1 -0
- package/dist/tests/unit/upload/run-upload.spec.js +43 -0
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -38,10 +38,11 @@ List commands and options: `npx @dataparade/cli --help` and `npx @dataparade/cli
|
|
|
38
38
|
|
|
39
39
|
## Upload to dashboard
|
|
40
40
|
|
|
41
|
-
After a scan, the CLI **auto-uploads** `dataflow.json`
|
|
41
|
+
After a scan, the CLI **auto-uploads** `dataflow.json` by default (unless you opt out). This creates an **import preview draft** in the web app — not a finished assessment.
|
|
42
42
|
|
|
43
|
+
- **With a workspace API key** (`DATAPARADE_WORKSPACE_API_KEY` from **Workspace → Access keys**): prints a dashboard link (`?importDraft=`) for that workspace.
|
|
44
|
+
- **Without a workspace key**: prints a sign-up link (`/preview/cli/<token>`). Create an account to open **Preview & Edit** on your dashboard.
|
|
43
45
|
- Opt out: `--skip-auto-upload` or `DATAPARADE_SKIP_AUTO_UPLOAD=true`
|
|
44
|
-
- Without a workspace key, the scan still succeeds; run `upload` later
|
|
45
46
|
- Upload alone does **not** consume scan quota
|
|
46
47
|
|
|
47
48
|
```bash
|
package/dist/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dataparade/cli",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.8",
|
|
4
4
|
"description": "dataPARADE CLI — scan codebases for privacy and security data flows",
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public",
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
},
|
|
9
9
|
"license": "GPL-3.0-or-later",
|
|
10
10
|
"author": "DataParade",
|
|
11
|
-
"homepage": "https://
|
|
11
|
+
"homepage": "https://dataparade.io",
|
|
12
12
|
"repository": {
|
|
13
13
|
"type": "git",
|
|
14
14
|
"url": "git+https://github.com/DataParade-io/dataparade-cli.git"
|
|
@@ -103,7 +103,10 @@ class ChatCompletionsFamilyProvider {
|
|
|
103
103
|
const body = {
|
|
104
104
|
model,
|
|
105
105
|
messages: [
|
|
106
|
-
{
|
|
106
|
+
{
|
|
107
|
+
role: "system",
|
|
108
|
+
content: request.systemPrompt ?? provider_enrichment_prompts_1.AI_PROVIDER_SYSTEM_PROMPT,
|
|
109
|
+
},
|
|
107
110
|
{ role: "user", content: request.prompt },
|
|
108
111
|
],
|
|
109
112
|
temperature: request.temperature ?? 0.1,
|
|
@@ -34,7 +34,9 @@ class GenerateContentFamilyProvider {
|
|
|
34
34
|
"content-type": "application/json",
|
|
35
35
|
},
|
|
36
36
|
body: JSON.stringify({
|
|
37
|
-
systemInstruction: {
|
|
37
|
+
systemInstruction: {
|
|
38
|
+
parts: [{ text: request.systemPrompt ?? provider_enrichment_prompts_1.AI_PROVIDER_SYSTEM_PROMPT }],
|
|
39
|
+
},
|
|
38
40
|
contents: [
|
|
39
41
|
{
|
|
40
42
|
role: "user",
|
|
@@ -33,7 +33,7 @@ class MessagesFamilyProvider {
|
|
|
33
33
|
model,
|
|
34
34
|
max_tokens: maxTokens,
|
|
35
35
|
temperature: request.temperature ?? 0.1,
|
|
36
|
-
system: (0, provider_enrichment_prompts_1.buildAnthropicEnrichmentSystemPrompt)(),
|
|
36
|
+
system: request.systemPrompt ?? (0, provider_enrichment_prompts_1.buildAnthropicEnrichmentSystemPrompt)(),
|
|
37
37
|
messages: [{ role: "user", content: request.prompt }],
|
|
38
38
|
}),
|
|
39
39
|
});
|
|
@@ -23,7 +23,7 @@ class OllamaGenerateFamilyProvider {
|
|
|
23
23
|
},
|
|
24
24
|
body: JSON.stringify({
|
|
25
25
|
model,
|
|
26
|
-
prompt: `${provider_enrichment_prompts_1.AI_PROVIDER_SYSTEM_PROMPT}\n\nUser payload:\n${request.prompt}`,
|
|
26
|
+
prompt: `${request.systemPrompt ?? provider_enrichment_prompts_1.AI_PROVIDER_SYSTEM_PROMPT}\n\nUser payload:\n${request.prompt}`,
|
|
27
27
|
stream: false,
|
|
28
28
|
options: {
|
|
29
29
|
temperature: request.temperature ?? 0.1,
|
|
@@ -6,6 +6,12 @@ export interface AiProviderRequest {
|
|
|
6
6
|
endpoint?: string;
|
|
7
7
|
temperature?: number;
|
|
8
8
|
maxTokens?: number;
|
|
9
|
+
/**
|
|
10
|
+
* Overrides the bundled enrichment system prompt (DP-P0-CLI-3813).
|
|
11
|
+
* Set by the DataParade backend on the Platform AI path so the server owns
|
|
12
|
+
* the system role. Unset for BYOK, which keeps the CLI-bundled prompt.
|
|
13
|
+
*/
|
|
14
|
+
systemPrompt?: string;
|
|
9
15
|
}
|
|
10
16
|
export interface AiProviderUsage {
|
|
11
17
|
inputTokens: number;
|
package/dist/src/cli.js
CHANGED
|
@@ -130,7 +130,8 @@ function createProgram() {
|
|
|
130
130
|
.option("--no-terraform-stack-section-auto", "Do not infer terraformStackSectionPathDepth from .tf layout when depth is unset")
|
|
131
131
|
.option("--monorepo-package-section-path-depth <n>", "Workspace package section depth (max 3): primary packages and rollup use N POSIX path segments (e.g. 3 for packages/twenty-apps/hello)")
|
|
132
132
|
.option("--no-monorepo-package-section-auto", "Do not infer monorepoPackageSectionPathDepth from package.json layout when depth is unset")
|
|
133
|
-
.option("--ai-inference", "Enable post-scan AI inference
|
|
133
|
+
.option("--ai-inference", "Enable post-scan AI inference (default: on; SCAN_AI_INFERENCE=false also disables)")
|
|
134
|
+
.option("--no-ai-inference", "Disable post-scan AI inference")
|
|
134
135
|
.option("--ai-provider <provider>", `AI provider: ${types_1.AI_PROVIDER_IDS.join("|")}`)
|
|
135
136
|
.option("--ai-model <model>", "Model name for AI inference")
|
|
136
137
|
.option("--ai-endpoint <url>", "Override provider endpoint URL")
|
|
@@ -154,6 +155,7 @@ function createProgram() {
|
|
|
154
155
|
let sentryScanRoot;
|
|
155
156
|
let sentryAiMode;
|
|
156
157
|
let sentryAiProvider;
|
|
158
|
+
const isInteractive = process.stdout.isTTY;
|
|
157
159
|
const workspaceApiKey = options.workspaceApiKey?.trim() ||
|
|
158
160
|
options.apiKey?.trim() ||
|
|
159
161
|
(0, scan_env_1.resolveWorkspaceApiKey)(process.env);
|
|
@@ -181,7 +183,6 @@ function createProgram() {
|
|
|
181
183
|
process.exitCode = 2;
|
|
182
184
|
return;
|
|
183
185
|
}
|
|
184
|
-
const isInteractive = process.stdout.isTTY;
|
|
185
186
|
const [{ scan, createDefaultScanConfiguration }, { buildDiagramGraphFromScanResult }] = await Promise.all([
|
|
186
187
|
Promise.resolve().then(() => __importStar(require("./core/pipeline/orchestrator"))),
|
|
187
188
|
Promise.resolve().then(() => __importStar(require("./core/pipeline/graph-mapping"))),
|
|
@@ -265,14 +266,53 @@ function createProgram() {
|
|
|
265
266
|
config.cliQuotaJobId = preflight.jobId;
|
|
266
267
|
if (config.enableAiInference &&
|
|
267
268
|
preflight.suggestedAiBudgetTokens > 0) {
|
|
268
|
-
|
|
269
|
+
// suggestedAiBudgetTokens is a server ceiling (remaining quota /
|
|
270
|
+
// per-job cap), not a spend target. Never raise the CLI working
|
|
271
|
+
// budget above the configured default (or user override).
|
|
272
|
+
config.aiBudgetTokens = Math.min(config.aiBudgetTokens ?? 12_000, preflight.suggestedAiBudgetTokens);
|
|
269
273
|
}
|
|
270
274
|
if (preflight.aiDelivery === "platform_proxy") {
|
|
271
275
|
config.aiMode = "platform";
|
|
272
276
|
}
|
|
273
277
|
if (isInteractive) {
|
|
274
278
|
// eslint-disable-next-line no-console
|
|
275
|
-
console.log(`[scan] quota: scans_remaining=${preflight.scansRemaining} ai_tokens_remaining=${preflight.aiTokensRemaining} job_id=${preflight.jobId}`);
|
|
279
|
+
console.log(`[scan] quota: scans_remaining=${preflight.scansRemaining} ai_tokens_remaining=${preflight.aiTokensRemaining} job_id=${preflight.jobId} working_budget=${config.aiBudgetTokens ?? 0}`);
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
else if (!workspaceApiKey &&
|
|
283
|
+
!usesByok &&
|
|
284
|
+
config.enableAiInference) {
|
|
285
|
+
try {
|
|
286
|
+
const { cliAnonymousAiSession } = await Promise.resolve().then(() => __importStar(require("./platform-api/anonymous-ai-session-client")));
|
|
287
|
+
const session = await cliAnonymousAiSession({
|
|
288
|
+
projectName: config.projectName,
|
|
289
|
+
});
|
|
290
|
+
cliQuotaJobId = session.jobId;
|
|
291
|
+
config.cliQuotaJobId = session.jobId;
|
|
292
|
+
config.anonSessionToken = session.sessionToken;
|
|
293
|
+
if (session.suggestedAiBudgetTokens > 0) {
|
|
294
|
+
// Anon suggested value is the hard per-job ceiling (default 100k).
|
|
295
|
+
// Applying it as the working budget caused scans to burn ~100k tokens.
|
|
296
|
+
config.aiBudgetTokens = Math.min(config.aiBudgetTokens ?? 12_000, session.suggestedAiBudgetTokens);
|
|
297
|
+
}
|
|
298
|
+
if (session.aiDelivery === "platform_proxy") {
|
|
299
|
+
config.aiMode = "platform";
|
|
300
|
+
}
|
|
301
|
+
if (isInteractive) {
|
|
302
|
+
// eslint-disable-next-line no-console
|
|
303
|
+
console.log(`[scan] anonymous platform AI: job_id=${session.jobId} working_budget=${config.aiBudgetTokens ?? 0} server_cap=${session.suggestedAiBudgetTokens} (claim debits actual tokens used)`);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
catch (anonSessionError) {
|
|
307
|
+
const { CliAnonymousIpLimitError } = await Promise.resolve().then(() => __importStar(require("./platform-api/anonymous-ai-session-client")));
|
|
308
|
+
const message = anonSessionError instanceof Error
|
|
309
|
+
? anonSessionError.message
|
|
310
|
+
: "Anonymous AI session failed.";
|
|
311
|
+
// eslint-disable-next-line no-console
|
|
312
|
+
console.error(`[scan] ${message}`);
|
|
313
|
+
process.exitCode =
|
|
314
|
+
anonSessionError instanceof CliAnonymousIpLimitError ? 2 : 1;
|
|
315
|
+
return;
|
|
276
316
|
}
|
|
277
317
|
}
|
|
278
318
|
const configValidation = (0, scan_config_schema_1.validateScanConfiguration)(config);
|
|
@@ -310,10 +350,12 @@ function createProgram() {
|
|
|
310
350
|
const s = scanResult.aiInferenceSummary;
|
|
311
351
|
// eslint-disable-next-line no-console
|
|
312
352
|
console.log(`[scan] llm-inference summary: candidates=${s.candidatesConsidered} proposals=${s.proposalsGenerated} (provider=${s.proposalsGeneratedProvider}) applied=${s.proposalsApplied} (provider=${s.proposalsAppliedProvider}) rejected=${s.proposalsRejected} (${s.aiProvider}/${s.aiModel})`);
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
353
|
+
// Always show real provider token usage (budget_cap is only a ceiling, not cost).
|
|
354
|
+
// eslint-disable-next-line no-console
|
|
355
|
+
console.log(`[scan] ai-usage: provider_calls=${s.providerCalls} tokens_in=${s.inputTokens} tokens_out=${s.outputTokens} tokens_total=${s.totalTokens}` +
|
|
356
|
+
(typeof s.estimatedCostUsd === "number"
|
|
357
|
+
? ` estimated_cost_usd=${formatUsd(s.estimatedCostUsd)}`
|
|
358
|
+
: ""));
|
|
317
359
|
if (options.aiVerbose && scanResult.aiInferenceProposalDetails) {
|
|
318
360
|
printAiInferenceVerbose(scanResult.aiInferenceProposalDetails);
|
|
319
361
|
}
|
|
@@ -387,30 +429,23 @@ function createProgram() {
|
|
|
387
429
|
const skipAutoUpload = Boolean(options.skipAutoUpload) ||
|
|
388
430
|
(0, upload_env_1.resolveSkipAutoUpload)(process.env);
|
|
389
431
|
if (!skipAutoUpload) {
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
432
|
+
try {
|
|
433
|
+
const { runDataflowUpload } = await Promise.resolve().then(() => __importStar(require("./upload/run-upload")));
|
|
434
|
+
const dataflowWrapper = (0, json_1.buildDataflowWrapper)(scanResult, diagramGraph, { projectName: resolvedProjectName });
|
|
435
|
+
await runDataflowUpload({
|
|
436
|
+
apiKey: workspaceApiKey,
|
|
437
|
+
dataflow: dataflowWrapper,
|
|
438
|
+
projectName: resolvedProjectName,
|
|
439
|
+
scanJobId: cliQuotaJobId,
|
|
440
|
+
logPrefix: "[scan]",
|
|
441
|
+
});
|
|
394
442
|
}
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
dataflow: dataflowWrapper,
|
|
402
|
-
projectName: resolvedProjectName,
|
|
403
|
-
scanJobId: cliQuotaJobId,
|
|
404
|
-
logPrefix: "[scan]",
|
|
405
|
-
});
|
|
406
|
-
}
|
|
407
|
-
catch (uploadError) {
|
|
408
|
-
const uploadMessage = uploadError instanceof Error
|
|
409
|
-
? uploadError.message
|
|
410
|
-
: "Unknown upload error.";
|
|
411
|
-
// eslint-disable-next-line no-console
|
|
412
|
-
console.error(`[scan] Auto-upload failed: ${uploadMessage}`);
|
|
413
|
-
}
|
|
443
|
+
catch (uploadError) {
|
|
444
|
+
const uploadMessage = uploadError instanceof Error
|
|
445
|
+
? uploadError.message
|
|
446
|
+
: "Unknown upload error.";
|
|
447
|
+
// eslint-disable-next-line no-console
|
|
448
|
+
console.error(`[scan] Auto-upload failed: ${uploadMessage}`);
|
|
414
449
|
}
|
|
415
450
|
}
|
|
416
451
|
}
|
|
@@ -447,26 +482,37 @@ function createProgram() {
|
|
|
447
482
|
failureMessage: exitFailed ? "CLI scan did not complete successfully" : undefined,
|
|
448
483
|
});
|
|
449
484
|
quotaCompletionReported = true;
|
|
485
|
+
if (isInteractive) {
|
|
486
|
+
// eslint-disable-next-line no-console
|
|
487
|
+
console.log("[scan] Scan finished — you can start a new scan now.");
|
|
488
|
+
}
|
|
450
489
|
}
|
|
451
490
|
}
|
|
452
491
|
catch (err) {
|
|
453
492
|
const message = err instanceof Error ? err.message : "Unknown error during scan.";
|
|
454
493
|
fallbackFailureMessage = message;
|
|
455
|
-
const { CliScanQuotaExceededError } = await Promise.resolve().then(() => __importStar(require("./platform-api/scan-quota-client")));
|
|
494
|
+
const { CliScanQuotaExceededError, CliScanAlreadyRunningError } = await Promise.resolve().then(() => __importStar(require("./platform-api/scan-quota-client")));
|
|
456
495
|
const quotaBlocked = err instanceof CliScanQuotaExceededError;
|
|
496
|
+
const scanInProgress = err instanceof CliScanAlreadyRunningError;
|
|
457
497
|
await (0, scan_sentry_1.reportScanCliError)({
|
|
458
498
|
error: err,
|
|
459
499
|
scanRoot: sentryScanRoot,
|
|
460
500
|
jobId: cliQuotaJobId,
|
|
461
501
|
aiMode: sentryAiMode,
|
|
462
502
|
aiProvider: sentryAiProvider,
|
|
463
|
-
failurePhase: quotaBlocked ? "preflight" : "scan_command",
|
|
464
|
-
failureCode: quotaBlocked
|
|
503
|
+
failurePhase: quotaBlocked || scanInProgress ? "preflight" : "scan_command",
|
|
504
|
+
failureCode: quotaBlocked
|
|
505
|
+
? "scan_quota_exceeded"
|
|
506
|
+
: scanInProgress
|
|
507
|
+
? "scan_already_running"
|
|
508
|
+
: "scan_exception",
|
|
465
509
|
});
|
|
466
510
|
// eslint-disable-next-line no-console
|
|
467
511
|
console.error(quotaBlocked
|
|
468
512
|
? `[scan] workspace quota: ${message}`
|
|
469
|
-
:
|
|
513
|
+
: scanInProgress
|
|
514
|
+
? `[scan] scan in progress: ${message}`
|
|
515
|
+
: `Scan failed: ${message}`);
|
|
470
516
|
process.exitCode = 1;
|
|
471
517
|
}
|
|
472
518
|
finally {
|
|
@@ -481,6 +527,10 @@ function createProgram() {
|
|
|
481
527
|
failureMessage: fallbackFailureMessage,
|
|
482
528
|
});
|
|
483
529
|
quotaCompletionReported = true;
|
|
530
|
+
if (isInteractive) {
|
|
531
|
+
// eslint-disable-next-line no-console
|
|
532
|
+
console.log("[scan] Scan finished — you can start a new scan now.");
|
|
533
|
+
}
|
|
484
534
|
}
|
|
485
535
|
catch {
|
|
486
536
|
// Quota report failure must not mask the original scan error.
|
|
@@ -498,12 +548,6 @@ function createProgram() {
|
|
|
498
548
|
const apiKey = options.workspaceApiKey?.trim() ||
|
|
499
549
|
options.apiKey?.trim() ||
|
|
500
550
|
(0, scan_env_1.resolveWorkspaceApiKey)(process.env);
|
|
501
|
-
if (!apiKey) {
|
|
502
|
-
// eslint-disable-next-line no-console
|
|
503
|
-
console.error("[upload] error: workspace API key is required (DATAPARADE_WORKSPACE_API_KEY or --workspace-api-key)");
|
|
504
|
-
process.exitCode = 1;
|
|
505
|
-
return;
|
|
506
|
-
}
|
|
507
551
|
const filePath = path_1.default.resolve(process.cwd(), file || "dataflow.json");
|
|
508
552
|
let dataflow;
|
|
509
553
|
try {
|
package/dist/src/config/env.js
CHANGED
|
@@ -22,10 +22,15 @@ function parseBoolean(value) {
|
|
|
22
22
|
if (value == null)
|
|
23
23
|
return undefined;
|
|
24
24
|
const normalized = value.trim().toLowerCase();
|
|
25
|
-
if (normalized === "true")
|
|
25
|
+
if (normalized === "true" || normalized === "1" || normalized === "yes" || normalized === "on") {
|
|
26
26
|
return true;
|
|
27
|
-
|
|
27
|
+
}
|
|
28
|
+
if (normalized === "false" ||
|
|
29
|
+
normalized === "0" ||
|
|
30
|
+
normalized === "no" ||
|
|
31
|
+
normalized === "off") {
|
|
28
32
|
return false;
|
|
33
|
+
}
|
|
29
34
|
return undefined;
|
|
30
35
|
}
|
|
31
36
|
function loadCliConfigEnv(env) {
|
|
@@ -3,6 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.findCliPackageRoot = findCliPackageRoot;
|
|
6
7
|
exports.loadCliDotenv = loadCliDotenv;
|
|
7
8
|
/**
|
|
8
9
|
* Loads `cli/.env` when not running under Jest.
|
|
@@ -16,9 +17,12 @@ const node_path_1 = __importDefault(require("node:path"));
|
|
|
16
17
|
const dotenv_1 = require("dotenv");
|
|
17
18
|
const CLI_PACKAGE_NAME = "@dataparade/cli";
|
|
18
19
|
const tried = { current: false };
|
|
20
|
+
/** @internal Exported for unit tests. */
|
|
19
21
|
function findCliPackageRoot(startDir) {
|
|
20
22
|
let dir = node_path_1.default.resolve(startDir);
|
|
21
23
|
const { root } = node_path_1.default.parse(dir);
|
|
24
|
+
/** Fallback when tsc emits package.json under dist/ but no outer root is found. */
|
|
25
|
+
let distPackageRoot;
|
|
22
26
|
while (dir !== root) {
|
|
23
27
|
const pkgPath = node_path_1.default.join(dir, "package.json");
|
|
24
28
|
if (node_fs_1.default.existsSync(pkgPath)) {
|
|
@@ -26,7 +30,12 @@ function findCliPackageRoot(startDir) {
|
|
|
26
30
|
const raw = node_fs_1.default.readFileSync(pkgPath, "utf8");
|
|
27
31
|
const pkg = JSON.parse(raw);
|
|
28
32
|
if (pkg.name === CLI_PACKAGE_NAME) {
|
|
29
|
-
|
|
33
|
+
if (node_path_1.default.basename(dir) === "dist") {
|
|
34
|
+
distPackageRoot = dir;
|
|
35
|
+
}
|
|
36
|
+
else {
|
|
37
|
+
return dir;
|
|
38
|
+
}
|
|
30
39
|
}
|
|
31
40
|
}
|
|
32
41
|
catch {
|
|
@@ -35,7 +44,7 @@ function findCliPackageRoot(startDir) {
|
|
|
35
44
|
}
|
|
36
45
|
dir = node_path_1.default.dirname(dir);
|
|
37
46
|
}
|
|
38
|
-
return
|
|
47
|
+
return distPackageRoot;
|
|
39
48
|
}
|
|
40
49
|
function loadCliDotenv() {
|
|
41
50
|
if (tried.current)
|
|
@@ -13,15 +13,20 @@ function validateAiInferenceCredentials(config) {
|
|
|
13
13
|
Boolean(config.aiModel?.trim()) &&
|
|
14
14
|
Boolean(config.aiApiKey?.trim());
|
|
15
15
|
const hasWorkspace = Boolean(config.workspaceApiKey?.trim());
|
|
16
|
-
|
|
17
|
-
|
|
16
|
+
const hasAnonSession = Boolean(config.anonSessionToken?.trim());
|
|
17
|
+
if (hasByok && (hasWorkspace || hasAnonSession)) {
|
|
18
|
+
errors.push("ai-inference: set either SCAN_BYOK_* (your LLM provider) or platform AI (DATAPARADE_WORKSPACE_API_KEY or anonymous session), not both");
|
|
18
19
|
return errors;
|
|
19
20
|
}
|
|
20
|
-
if (
|
|
21
|
-
errors.push("ai-inference:
|
|
21
|
+
if (hasWorkspace && hasAnonSession) {
|
|
22
|
+
errors.push("ai-inference: set either DATAPARADE_WORKSPACE_API_KEY or an anonymous AI session, not both");
|
|
23
|
+
return errors;
|
|
24
|
+
}
|
|
25
|
+
if (!hasByok && !hasWorkspace && !hasAnonSession) {
|
|
26
|
+
errors.push("ai-inference: LLM inference requires SCAN_BYOK_PROVIDER, SCAN_BYOK_MODEL, and SCAN_BYOK_API_KEY, DATAPARADE_WORKSPACE_API_KEY for platform AI, or an anonymous platform AI session");
|
|
22
27
|
}
|
|
23
|
-
if (hasWorkspace && !config.cliQuotaJobId) {
|
|
24
|
-
errors.push("ai-inference: platform AI requires a quota
|
|
28
|
+
if ((hasWorkspace || hasAnonSession) && !config.cliQuotaJobId) {
|
|
29
|
+
errors.push("ai-inference: platform AI requires a quota/session job id (preflight or anonymous-session must succeed)");
|
|
25
30
|
}
|
|
26
31
|
return errors;
|
|
27
32
|
}
|
|
@@ -32,7 +37,8 @@ function resolveAiMode(config) {
|
|
|
32
37
|
return "hosted_worker";
|
|
33
38
|
if (config.aiApiKey?.trim())
|
|
34
39
|
return "byok";
|
|
35
|
-
if (config.workspaceApiKey?.trim())
|
|
40
|
+
if (config.workspaceApiKey?.trim() || config.anonSessionToken?.trim()) {
|
|
36
41
|
return "platform";
|
|
42
|
+
}
|
|
37
43
|
return "none";
|
|
38
44
|
}
|
|
@@ -8,7 +8,9 @@ function buildAgentOrchestratorOptions(config, opts) {
|
|
|
8
8
|
const platformProvider = process.env.SCAN_WORKER_LLM_PROVIDER?.trim() ||
|
|
9
9
|
"openai";
|
|
10
10
|
const platformModel = process.env.SCAN_WORKER_LLM_MODEL?.trim() ||
|
|
11
|
-
config.aiModel?.trim()
|
|
11
|
+
(config.aiModel?.trim() && config.aiModel.trim() !== "heuristic"
|
|
12
|
+
? config.aiModel.trim()
|
|
13
|
+
: undefined) ||
|
|
12
14
|
"gpt-4o-mini";
|
|
13
15
|
const base = {
|
|
14
16
|
provider: aiMode === "platform"
|
|
@@ -29,11 +31,13 @@ function buildAgentOrchestratorOptions(config, opts) {
|
|
|
29
31
|
skipStructuralHeuristics: opts.skipStructuralHeuristics,
|
|
30
32
|
};
|
|
31
33
|
if (aiMode === "platform" &&
|
|
32
|
-
config.
|
|
33
|
-
config.
|
|
34
|
+
config.cliQuotaJobId?.trim() &&
|
|
35
|
+
(config.workspaceApiKey?.trim() || config.anonSessionToken?.trim())) {
|
|
34
36
|
base.platformProxy = {
|
|
35
37
|
apiBaseUrl: config.platformApiBaseUrl ?? (0, dataparade_api_base_url_1.getDataparadeApiBaseUrl)(),
|
|
36
|
-
workspaceApiKey: config.workspaceApiKey
|
|
38
|
+
workspaceApiKey: (config.workspaceApiKey?.trim() ||
|
|
39
|
+
config.anonSessionToken?.trim() ||
|
|
40
|
+
""),
|
|
37
41
|
jobId: config.cliQuotaJobId.trim(),
|
|
38
42
|
};
|
|
39
43
|
base.apiKey = undefined;
|
|
@@ -27,7 +27,7 @@ function createDefaultScanConfiguration(overrides = {}) {
|
|
|
27
27
|
languages: undefined,
|
|
28
28
|
minimumConfidence: 0.5,
|
|
29
29
|
deepAnalysis: false,
|
|
30
|
-
enableAiInference:
|
|
30
|
+
enableAiInference: true,
|
|
31
31
|
aiProvider: "mock",
|
|
32
32
|
aiModel: "heuristic",
|
|
33
33
|
aiEndpoint: undefined,
|
|
@@ -48,6 +48,7 @@ export declare const scanConfigurationSchema: z.ZodObject<{
|
|
|
48
48
|
}>>;
|
|
49
49
|
aiVerbose: z.ZodOptional<z.ZodBoolean>;
|
|
50
50
|
workspaceApiKey: z.ZodOptional<z.ZodString>;
|
|
51
|
+
anonSessionToken: z.ZodOptional<z.ZodString>;
|
|
51
52
|
aiMode: z.ZodOptional<z.ZodEnum<{
|
|
52
53
|
byok: "byok";
|
|
53
54
|
platform: "platform";
|
|
@@ -45,6 +45,7 @@ exports.scanConfigurationSchema = zod_1.z.object({
|
|
|
45
45
|
aiInferenceScope: zod_1.z.enum(["default", "third_party_only"]).optional(),
|
|
46
46
|
aiVerbose: zod_1.z.boolean().optional(),
|
|
47
47
|
workspaceApiKey: zod_1.z.string().min(1).optional(),
|
|
48
|
+
anonSessionToken: zod_1.z.string().min(1).optional(),
|
|
48
49
|
aiMode: zod_1.z.enum(["byok", "platform", "hosted_worker", "none"]).optional(),
|
|
49
50
|
platformApiBaseUrl: zod_1.z.string().min(1).optional(),
|
|
50
51
|
cliQuotaJobId: zod_1.z.string().min(1).optional(),
|
|
@@ -48,6 +48,11 @@ export interface ScanConfiguration {
|
|
|
48
48
|
aiThirdPartyDataFlowEnabled?: boolean;
|
|
49
49
|
/** DataParade workspace API key (quota + platform LLM proxy). */
|
|
50
50
|
workspaceApiKey?: string;
|
|
51
|
+
/**
|
|
52
|
+
* Anonymous platform AI session token (`dp_anon_…`) from
|
|
53
|
+
* POST /api/scans/cli/ai/anonymous-session. Mutually exclusive with workspaceApiKey.
|
|
54
|
+
*/
|
|
55
|
+
anonSessionToken?: string;
|
|
51
56
|
/** How LLM inference is billed: byok (direct provider) or platform (API proxy). */
|
|
52
57
|
aiMode?: "byok" | "platform" | "hosted_worker" | "none";
|
|
53
58
|
/** Base URL for platform API (preflight / infer / complete). */
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export type AnonymousAiSessionResponse = {
|
|
2
|
+
sessionToken: string;
|
|
3
|
+
jobId: string;
|
|
4
|
+
suggestedAiBudgetTokens: number;
|
|
5
|
+
expiresAt: string;
|
|
6
|
+
aiDelivery: "platform_proxy" | "none";
|
|
7
|
+
};
|
|
8
|
+
export declare class CliAnonymousIpLimitError extends Error {
|
|
9
|
+
readonly status = 429;
|
|
10
|
+
constructor(message: string);
|
|
11
|
+
}
|
|
12
|
+
export declare function cliAnonymousAiSession(input?: {
|
|
13
|
+
projectName?: string;
|
|
14
|
+
}): Promise<AnonymousAiSessionResponse>;
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.CliAnonymousIpLimitError = void 0;
|
|
4
|
+
exports.cliAnonymousAiSession = cliAnonymousAiSession;
|
|
5
|
+
const dataparade_api_base_url_1 = require("./dataparade-api-base-url");
|
|
6
|
+
class CliAnonymousIpLimitError extends Error {
|
|
7
|
+
constructor(message) {
|
|
8
|
+
super(message);
|
|
9
|
+
this.status = 429;
|
|
10
|
+
this.name = "CliAnonymousIpLimitError";
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
exports.CliAnonymousIpLimitError = CliAnonymousIpLimitError;
|
|
14
|
+
async function cliAnonymousAiSession(input) {
|
|
15
|
+
const res = await fetch(`${(0, dataparade_api_base_url_1.getDataparadeApiBaseUrl)()}/api/scans/cli/ai/anonymous-session`, {
|
|
16
|
+
method: "POST",
|
|
17
|
+
headers: { "Content-Type": "application/json" },
|
|
18
|
+
body: JSON.stringify({
|
|
19
|
+
projectName: input?.projectName,
|
|
20
|
+
}),
|
|
21
|
+
});
|
|
22
|
+
if (!res.ok) {
|
|
23
|
+
let message = `Anonymous AI session failed (${res.status})`;
|
|
24
|
+
try {
|
|
25
|
+
const body = (await res.json());
|
|
26
|
+
if (typeof body.message === "string")
|
|
27
|
+
message = body.message;
|
|
28
|
+
else if (Array.isArray(body.message))
|
|
29
|
+
message = body.message.join(", ");
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
// ignore
|
|
33
|
+
}
|
|
34
|
+
if (res.status === 429) {
|
|
35
|
+
throw new CliAnonymousIpLimitError(message);
|
|
36
|
+
}
|
|
37
|
+
throw new Error(message);
|
|
38
|
+
}
|
|
39
|
+
return (await res.json());
|
|
40
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
type NestedApiError = {
|
|
2
|
+
code?: string;
|
|
3
|
+
message?: string;
|
|
4
|
+
};
|
|
5
|
+
export type CliApiErrorBody = {
|
|
6
|
+
code?: string;
|
|
7
|
+
message?: string | string[] | NestedApiError;
|
|
8
|
+
statusCode?: number;
|
|
9
|
+
};
|
|
10
|
+
export declare function parseCliApiErrorBody(body: unknown): {
|
|
11
|
+
code?: string;
|
|
12
|
+
message?: string;
|
|
13
|
+
};
|
|
14
|
+
export declare function readCliApiErrorMessage(body: unknown, fallback: string): {
|
|
15
|
+
code?: string;
|
|
16
|
+
message: string;
|
|
17
|
+
};
|
|
18
|
+
export {};
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.parseCliApiErrorBody = parseCliApiErrorBody;
|
|
4
|
+
exports.readCliApiErrorMessage = readCliApiErrorMessage;
|
|
5
|
+
function parseCliApiErrorBody(body) {
|
|
6
|
+
if (!body || typeof body !== "object") {
|
|
7
|
+
return {};
|
|
8
|
+
}
|
|
9
|
+
const parsed = body;
|
|
10
|
+
if (typeof parsed.message === "string") {
|
|
11
|
+
return { code: parsed.code, message: parsed.message };
|
|
12
|
+
}
|
|
13
|
+
if (Array.isArray(parsed.message)) {
|
|
14
|
+
return { code: parsed.code, message: parsed.message.join(", ") };
|
|
15
|
+
}
|
|
16
|
+
if (parsed.message && typeof parsed.message === "object") {
|
|
17
|
+
const nested = parsed.message;
|
|
18
|
+
return {
|
|
19
|
+
code: nested.code ?? parsed.code,
|
|
20
|
+
message: typeof nested.message === "string" ? nested.message : undefined,
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
return { code: parsed.code };
|
|
24
|
+
}
|
|
25
|
+
function readCliApiErrorMessage(body, fallback) {
|
|
26
|
+
const parsed = parseCliApiErrorBody(body);
|
|
27
|
+
return {
|
|
28
|
+
code: parsed.code,
|
|
29
|
+
message: parsed.message ?? fallback,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
@@ -3,6 +3,11 @@ export declare class CliScanQuotaExceededError extends Error {
|
|
|
3
3
|
readonly code: "scan_quota_exceeded";
|
|
4
4
|
constructor(message: string);
|
|
5
5
|
}
|
|
6
|
+
/** Thrown when preflight is rejected with HTTP 409 scan_already_running. */
|
|
7
|
+
export declare class CliScanAlreadyRunningError extends Error {
|
|
8
|
+
readonly code: "scan_already_running";
|
|
9
|
+
constructor(message: string);
|
|
10
|
+
}
|
|
6
11
|
export type CliScanPreflightResponse = {
|
|
7
12
|
allowed: boolean;
|
|
8
13
|
jobId: string;
|