@prism-d1/cli 1.0.79 → 1.0.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/github-workflows/prism-ai-metrics.yml +83 -122
- package/dist/assets/github-workflows/prism-eval-gate-kiro.yml +66 -0
- package/dist/assets/github-workflows/prism-eval-gate.yml +1 -1
- package/dist/assets/gitlab-workflows/.gitlab-ci.yml +0 -3
- package/dist/assets/gitlab-workflows/prism-ai-metrics.yml +53 -56
- package/dist/assets/infra/bin/app.ts +1 -0
- package/dist/assets/infra/lib/constructs/otel-collector-construct.ts +4 -0
- package/dist/assets/infra/lib/dashboard-stack.ts +378 -1235
- package/dist/assets/infra/lib/lambda/event-schema.ts +144 -0
- package/dist/assets/infra/lib/lambda/metrics-processor.ts +124 -36
- package/dist/assets/infra/lib/lambda/otel-receiver.ts +249 -35
- package/dist/assets/infra/lib/lambda/security-remediation-tracker.ts +17 -2
- package/dist/assets/infra/lib/lambda/velocity-widget.ts +1483 -0
- package/dist/assets/infra/lib/metrics-pipeline-stack.ts +111 -119
- package/dist/assets/infra/package.json +2 -1
- package/dist/assets/infra/tsconfig.json +9 -2
- package/dist/src/commands/assessment/web.d.ts.map +1 -1
- package/dist/src/commands/assessment/web.js +25 -2
- package/dist/src/commands/assessment/web.js.map +1 -1
- package/dist/src/commands/bootstrapper/install-eval-harness.d.ts +8 -2
- package/dist/src/commands/bootstrapper/install-eval-harness.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/install-eval-harness.js +46 -36
- package/dist/src/commands/bootstrapper/install-eval-harness.js.map +1 -1
- package/dist/src/commands/bootstrapper/install-git-hooks.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/install-git-hooks.js +10 -14
- package/dist/src/commands/bootstrapper/install-git-hooks.js.map +1 -1
- package/dist/src/commands/bootstrapper/install-github-workflows.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/install-github-workflows.js +7 -2
- package/dist/src/commands/bootstrapper/install-github-workflows.js.map +1 -1
- package/dist/src/commands/bootstrapper/install-gitlab-workflows.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/install-gitlab-workflows.js +7 -1
- package/dist/src/commands/bootstrapper/install-gitlab-workflows.js.map +1 -1
- package/dist/src/commands/bootstrapper/setup-github-oidc.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/setup-github-oidc.js +50 -27
- package/dist/src/commands/bootstrapper/setup-github-oidc.js.map +1 -1
- package/dist/src/commands/bootstrapper/setup-gitlab-oidc.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/setup-gitlab-oidc.js +69 -25
- package/dist/src/commands/bootstrapper/setup-gitlab-oidc.js.map +1 -1
- package/dist/src/commands/bootstrapper/setup-otel-sync.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/setup-otel-sync.js +145 -53
- package/dist/src/commands/bootstrapper/setup-otel-sync.js.map +1 -1
- package/dist/src/commands/workshop/generate-demo-data.d.ts +41 -2
- package/dist/src/commands/workshop/generate-demo-data.d.ts.map +1 -1
- package/dist/src/commands/workshop/generate-demo-data.js +391 -167
- package/dist/src/commands/workshop/generate-demo-data.js.map +1 -1
- package/dist/src/commands/workshop/verify-setup.d.ts.map +1 -1
- package/dist/src/commands/workshop/verify-setup.js +75 -14
- package/dist/src/commands/workshop/verify-setup.js.map +1 -1
- package/dist/src/utils/exec.d.ts +56 -0
- package/dist/src/utils/exec.d.ts.map +1 -0
- package/dist/src/utils/exec.js +72 -0
- package/dist/src/utils/exec.js.map +1 -0
- package/dist/src/utils/region.d.ts +50 -0
- package/dist/src/utils/region.d.ts.map +1 -0
- package/dist/src/utils/region.js +62 -0
- package/dist/src/utils/region.js.map +1 -0
- package/dist/src/utils/tempfile.d.ts +19 -0
- package/dist/src/utils/tempfile.d.ts.map +1 -0
- package/dist/src/utils/tempfile.js +32 -0
- package/dist/src/utils/tempfile.js.map +1 -0
- package/dist/src/utils/validate.d.ts +54 -0
- package/dist/src/utils/validate.d.ts.map +1 -0
- package/dist/src/utils/validate.js +143 -0
- package/dist/src/utils/validate.js.map +1 -0
- package/package.json +2 -2
- package/dist/assets/eval-harness/README.md +0 -144
- package/dist/assets/github-workflows/README.md +0 -110
- package/dist/assets/github-workflows/prism-dora-weekly.yml +0 -334
- package/dist/assets/gitlab-workflows/prism-dora-weekly.yml +0 -181
- package/dist/assets/infra/lib/lambda/defect-correlator.ts +0 -142
- package/dist/assets/infra/lib/lambda/spec-to-code-calculator.ts +0 -123
- package/dist/assets/metric-hooks/README.md +0 -107
|
@@ -1,60 +1,189 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
1
|
+
/**
|
|
2
|
+
* PRISM D1 — workshop demo data generator.
|
|
3
|
+
*
|
|
4
|
+
* The dashboards read TWO stores, and a generator that fills only one leaves
|
|
5
|
+
* roughly half of every view blank:
|
|
6
|
+
*
|
|
7
|
+
* EventBridge → metrics-processor → events table + CloudWatch
|
|
8
|
+
* Delivery KPIs, eval gates, governance, agent ops, security, most of CISO.
|
|
9
|
+
*
|
|
10
|
+
* AI-usage table (attribution) → otel-receiver GET routes
|
|
11
|
+
* AI share, merge rate, defect rate, $/shipped commit, Attribution
|
|
12
|
+
* Coverage, Repository Breakdown, the observed PRISM level, and the whole
|
|
13
|
+
* Developer Productivity dashboard.
|
|
14
|
+
*
|
|
15
|
+
* Attribution is written directly to DynamoDB rather than POSTed to
|
|
16
|
+
* /v1/traces because the receiver derives the owning developer from the
|
|
17
|
+
* caller's JWT claim. Every seeded commit would land under the single
|
|
18
|
+
* participant running this command, and a one-developer fleet cannot
|
|
19
|
+
* demonstrate the per-developer view or a coverage shortfall. Direct writes
|
|
20
|
+
* are the only way to seed a multi-developer fleet.
|
|
21
|
+
*
|
|
22
|
+
* The cost of that choice is schema coupling. Item shapes below must match
|
|
23
|
+
* infra/lib/lambda/otel-receiver.ts — specifically writeCommitAttribution,
|
|
24
|
+
* writeSpanIfNew and bumpDailyAggregate. If those change, this changes too.
|
|
25
|
+
*
|
|
26
|
+
* Guardrails, because this writes fabricated facts into a metrics store:
|
|
27
|
+
* - every seeded item carries demo_seed=true and a seed_run_id
|
|
28
|
+
* - seeding refuses to run when the table already holds real attribution,
|
|
29
|
+
* unless --force is passed
|
|
30
|
+
* - --purge-demo removes seeded items and leaves real ones alone
|
|
31
|
+
*/
|
|
32
|
+
import { randomBytes } from 'node:crypto';
|
|
33
|
+
import { run } from '../../utils/exec.js';
|
|
34
|
+
import { withPrivateTemp } from '../../utils/tempfile.js';
|
|
35
|
+
// ---- Shell-free AWS CLI helpers ----
|
|
36
|
+
/**
|
|
37
|
+
* EventBridge put-events with the entry array passed as a file.
|
|
38
|
+
*
|
|
39
|
+
* The previous form interpolated JSON.stringify output into a single-quoted
|
|
40
|
+
* shell string (`--entries '${json}'`). A team, bus or repo name containing an
|
|
41
|
+
* apostrophe terminated that quoted region and the remainder became shell
|
|
42
|
+
* source. Passing a file path removes both the quoting and the shell.
|
|
43
|
+
*/
|
|
44
|
+
function putEvents(region, entries) {
|
|
45
|
+
return withPrivateTemp('entries.json', JSON.stringify(entries), (file) => run('aws', ['events', 'put-events', '--region', region, '--entries', `file://${file}`]).ok);
|
|
46
|
+
}
|
|
47
|
+
/** BatchWriteItem in DynamoDB's 25-request chunks. Returns items written. */
|
|
48
|
+
function ddbBatchWrite(region, table, requests) {
|
|
49
|
+
let written = 0;
|
|
50
|
+
for (let i = 0; i < requests.length; i += 25) {
|
|
51
|
+
const chunk = requests.slice(i, i + 25);
|
|
52
|
+
const payload = JSON.stringify({ [table]: chunk });
|
|
53
|
+
const ok = withPrivateTemp('batch.json', payload, (file) => run('aws', ['dynamodb', 'batch-write-item', '--region', region, '--request-items', `file://${file}`]).ok);
|
|
54
|
+
if (ok) {
|
|
55
|
+
written += chunk.length;
|
|
56
|
+
process.stdout.write('.');
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
console.error(`\nWarning: failed to write a batch of ${chunk.length} attribution items`);
|
|
60
|
+
}
|
|
9
61
|
}
|
|
62
|
+
return written;
|
|
10
63
|
}
|
|
11
|
-
|
|
64
|
+
/** True when the table holds attribution commits that this generator did not create. */
|
|
65
|
+
function hasRealAttribution(region, table) {
|
|
66
|
+
const values = JSON.stringify({ ':rt': { S: 'OTEL_ATTR_COMMIT' } });
|
|
67
|
+
const result = withPrivateTemp('values.json', values, (file) => run('aws', [
|
|
68
|
+
'dynamodb', 'scan',
|
|
69
|
+
'--region', region,
|
|
70
|
+
'--table-name', table,
|
|
71
|
+
'--select', 'COUNT',
|
|
72
|
+
'--filter-expression', 'record_type = :rt AND attribute_not_exists(demo_seed)',
|
|
73
|
+
'--expression-attribute-values', `file://${file}`,
|
|
74
|
+
'--output', 'json',
|
|
75
|
+
]));
|
|
76
|
+
if (!result.ok)
|
|
77
|
+
return false; // table missing or no access — surfaced by the caller's first write
|
|
12
78
|
try {
|
|
13
|
-
return
|
|
79
|
+
return (JSON.parse(result.stdout).Count ?? 0) > 0;
|
|
14
80
|
}
|
|
15
81
|
catch {
|
|
16
|
-
return
|
|
82
|
+
return false;
|
|
17
83
|
}
|
|
18
84
|
}
|
|
19
|
-
function
|
|
85
|
+
function purgeDemoAttribution(region, table) {
|
|
86
|
+
const values = JSON.stringify({ ':t': { BOOL: true } });
|
|
87
|
+
const result = withPrivateTemp('values.json', values, (file) => run('aws', [
|
|
88
|
+
'dynamodb', 'scan',
|
|
89
|
+
'--region', region,
|
|
90
|
+
'--table-name', table,
|
|
91
|
+
'--filter-expression', 'demo_seed = :t',
|
|
92
|
+
'--projection-expression', 'pk,sk',
|
|
93
|
+
'--expression-attribute-values', `file://${file}`,
|
|
94
|
+
'--output', 'json',
|
|
95
|
+
]));
|
|
96
|
+
if (!result.ok) {
|
|
97
|
+
console.error(`Error: could not scan ${table}: ${result.stderr}`);
|
|
98
|
+
return 0;
|
|
99
|
+
}
|
|
100
|
+
let items;
|
|
20
101
|
try {
|
|
21
|
-
|
|
22
|
-
return true;
|
|
102
|
+
items = JSON.parse(result.stdout).Items ?? [];
|
|
23
103
|
}
|
|
24
104
|
catch {
|
|
25
|
-
|
|
105
|
+
console.error('Error: could not parse scan output');
|
|
106
|
+
return 0;
|
|
26
107
|
}
|
|
108
|
+
if (items.length === 0)
|
|
109
|
+
return 0;
|
|
110
|
+
return ddbBatchWrite(region, table, items.map(key => ({ DeleteRequest: { Key: key } })));
|
|
27
111
|
}
|
|
112
|
+
// ---- Synthetic fleet ----
|
|
113
|
+
const SONNET = 'us.anthropic.claude-sonnet-4-5-20250929-v1:0';
|
|
114
|
+
const HAIKU = 'us.anthropic.claude-haiku-4-5-20251001-v1:0';
|
|
115
|
+
/**
|
|
116
|
+
* `onboarded: false` is deliberate and load-bearing. That developer authors
|
|
117
|
+
* PRs, so their commits appear in the CI census that forms the Attribution
|
|
118
|
+
* Coverage denominator, but they produce no attribution items. Coverage
|
|
119
|
+
* therefore lands below 100% for the honest reason — someone has not run
|
|
120
|
+
* setup-otel-sync — which is exactly what the KPI exists to reveal. Seeding
|
|
121
|
+
* everyone would render a permanent 100% and teach the opposite lesson.
|
|
122
|
+
*/
|
|
123
|
+
const DEVELOPERS = [
|
|
124
|
+
{ email: 'alex.rivera@example.com', tool: 'claude-code', model: SONNET, aiRate: 0.74, onboarded: true },
|
|
125
|
+
{ email: 'priya.natarajan@example.com', tool: 'kiro', model: HAIKU, aiRate: 0.61, onboarded: true },
|
|
126
|
+
{ email: 'sam.okafor@example.com', tool: 'claude-code', model: SONNET, aiRate: 0.83, onboarded: true },
|
|
127
|
+
{ email: 'wei.zhang@example.com', tool: 'cursor', model: HAIKU, aiRate: 0.42, onboarded: true },
|
|
128
|
+
{ email: 'jordan.blake@example.com', tool: 'none', model: '', aiRate: 0, onboarded: false },
|
|
129
|
+
];
|
|
130
|
+
const SPAN_TTL_DAYS = 90;
|
|
131
|
+
const COMMIT_TTL_DAYS = 365;
|
|
28
132
|
function randomInt(min, max) {
|
|
29
133
|
return Math.floor(Math.random() * (max - min + 1)) + min;
|
|
30
134
|
}
|
|
31
135
|
function formatDate(date) {
|
|
32
136
|
return date.toISOString().split('T')[0];
|
|
33
137
|
}
|
|
138
|
+
function hex(bytes) {
|
|
139
|
+
return randomBytes(bytes).toString('hex');
|
|
140
|
+
}
|
|
34
141
|
export default {
|
|
35
|
-
description: 'Generate sample
|
|
142
|
+
description: 'Generate sample data for the PRISM dashboards (8 days of events + attribution)',
|
|
36
143
|
options: [
|
|
37
144
|
{ flags: '--region <region>', description: 'AWS region', default: 'us-west-2' },
|
|
38
145
|
{ flags: '--bus <name>', description: 'EventBridge bus name', default: 'prism-d1-metrics' },
|
|
39
146
|
{ flags: '--team <id>', description: 'Team identifier', default: 'demo-team' },
|
|
40
|
-
{ flags: '--repo <name>', description: '
|
|
147
|
+
{ flags: '--repo <name>', description: 'Primary repository name', default: 'prism-d1-sample-app' },
|
|
148
|
+
{ flags: '--table <name>', description: 'Attribution (AI usage) table name', default: 'prism-d1-ai-usage' },
|
|
149
|
+
{ flags: '--no-attribution', description: 'Emit events only; leave the attribution store untouched' },
|
|
150
|
+
{ flags: '--purge-demo', description: 'Delete previously seeded attribution items and exit' },
|
|
151
|
+
{ flags: '--force', description: 'Seed attribution even when the table already holds real data' },
|
|
41
152
|
],
|
|
42
153
|
action(options) {
|
|
43
|
-
const { region, bus, team, repo } = options;
|
|
44
|
-
|
|
45
|
-
|
|
154
|
+
const { region, bus, team, repo, table } = options;
|
|
155
|
+
const seedAttribution = options.attribution !== false;
|
|
156
|
+
// No shell here, so `command -v aws` is not available — `command` is a
|
|
157
|
+
// shell builtin, not a binary. Probe the binary directly instead.
|
|
158
|
+
if (!run('aws', ['--version']).ok) {
|
|
46
159
|
console.error('Error: AWS CLI not found. Install from https://aws.amazon.com/cli/');
|
|
47
160
|
process.exit(1);
|
|
48
161
|
}
|
|
49
|
-
if (
|
|
50
|
-
console.
|
|
51
|
-
|
|
162
|
+
if (options.purgeDemo) {
|
|
163
|
+
console.log(`Purging seeded attribution items from ${table} (${region})...`);
|
|
164
|
+
const removed = purgeDemoAttribution(region, table);
|
|
165
|
+
console.log(`\nRemoved ${removed} seeded item(s). Real attribution data was left untouched.`);
|
|
166
|
+
return;
|
|
52
167
|
}
|
|
53
168
|
console.log('=== PRISM Demo Data Generator ===');
|
|
54
169
|
console.log(`Region: ${region} | Bus: ${bus} | Team: ${team}`);
|
|
170
|
+
console.log(`Attribution: ${seedAttribution ? `${table}` : 'skipped (--no-attribution)'}`);
|
|
55
171
|
console.log('');
|
|
56
|
-
|
|
57
|
-
|
|
172
|
+
if (seedAttribution && !options.force && hasRealAttribution(region, table)) {
|
|
173
|
+
console.error(`Error: ${table} already contains real attribution data.`);
|
|
174
|
+
console.error('');
|
|
175
|
+
console.error('Seeding fabricated commits alongside real developer activity corrupts the');
|
|
176
|
+
console.error('very KPIs this dashboard exists to report — AI share, coverage, $/shipped.');
|
|
177
|
+
console.error('');
|
|
178
|
+
console.error(' --no-attribution emit events only, leave attribution alone');
|
|
179
|
+
console.error(' --force seed anyway (marked demo_seed=true, removable)');
|
|
180
|
+
process.exit(1);
|
|
181
|
+
}
|
|
182
|
+
// A repo set, so the Repository Breakdown view has more than one row. The
|
|
183
|
+
// per-day PR count is distributed across these rather than multiplied by
|
|
184
|
+
// them, so total event volume is unchanged.
|
|
185
|
+
const REPOS = [repo, `${repo}-api`, `${repo}-web`];
|
|
186
|
+
const seedRunId = hex(8);
|
|
58
187
|
let batch = [];
|
|
59
188
|
let total = 0;
|
|
60
189
|
function flush() {
|
|
@@ -80,66 +209,171 @@ export default {
|
|
|
80
209
|
process.stdout.write('.');
|
|
81
210
|
}
|
|
82
211
|
}
|
|
212
|
+
// ---- Attribution accumulators ----
|
|
213
|
+
//
|
|
214
|
+
// Spans and commits are collected SEPARATELY because write order matters.
|
|
215
|
+
// Each COMMIT# put fires a DynamoDB stream event into
|
|
216
|
+
// attribution-metrics-publisher, which resolves origin by querying the
|
|
217
|
+
// user's SPAN# items for a matching trace_id — it ignores the stored
|
|
218
|
+
// ai_origin verdict. If a commit's stream event is processed before its
|
|
219
|
+
// span exists, the join misses and an AI commit publishes as HumanCommits.
|
|
220
|
+
// The receiver avoids this by running processSpans before
|
|
221
|
+
// processAttributionSpans; this mirrors that ordering.
|
|
222
|
+
const spanRequests = [];
|
|
223
|
+
const commitRequests = [];
|
|
224
|
+
/** (user, day, tool, model) → running totals for the OTEL#DAY rollup. */
|
|
225
|
+
const dayRollups = new Map();
|
|
226
|
+
const commitTtl = Math.floor(Date.now() / 1000) + COMMIT_TTL_DAYS * 86400;
|
|
227
|
+
const spanTtl = Math.floor(Date.now() / 1000) + SPAN_TTL_DAYS * 86400;
|
|
228
|
+
/**
|
|
229
|
+
* Seed one commit's attribution facts.
|
|
230
|
+
*
|
|
231
|
+
* `ai_origin` is written explicitly here because the receiver freezes it at
|
|
232
|
+
* ingest and every read path prefers the stored verdict. The SPAN# item is
|
|
233
|
+
* NOT optional for AI commits: attribution-metrics-publisher ignores the
|
|
234
|
+
* stored verdict and re-derives origin by joining traceId against spans, so
|
|
235
|
+
* an AI commit with no matching span publishes as HumanCommits. Omitting
|
|
236
|
+
* spans would make the native CloudWatch graphs contradict the custom
|
|
237
|
+
* widgets on the same dashboard.
|
|
238
|
+
*/
|
|
239
|
+
function seedCommit(dev, repoName, sha, timestamp, isAi, inMain, wasReverted) {
|
|
240
|
+
const traceId = hex(16);
|
|
241
|
+
const sessionId = `sess-${seedRunId}-${hex(4)}`;
|
|
242
|
+
commitRequests.push({
|
|
243
|
+
PutRequest: {
|
|
244
|
+
Item: {
|
|
245
|
+
pk: { S: `REPO#${repoName}` },
|
|
246
|
+
sk: { S: `COMMIT#${sha}` },
|
|
247
|
+
record_type: { S: 'OTEL_ATTR_COMMIT' },
|
|
248
|
+
session_id: { S: sessionId },
|
|
249
|
+
trace_id: { S: traceId },
|
|
250
|
+
project: { S: repoName },
|
|
251
|
+
user: { S: dev.email },
|
|
252
|
+
device_id: { S: `demo-${seedRunId}` },
|
|
253
|
+
in_main: { BOOL: inMain },
|
|
254
|
+
was_reverted: { BOOL: wasReverted },
|
|
255
|
+
timestamp: { S: timestamp },
|
|
256
|
+
updated_at: { S: new Date().toISOString() },
|
|
257
|
+
ttl: { N: String(commitTtl) },
|
|
258
|
+
ai_origin: { S: isAi ? 'ai-generated' : 'human' },
|
|
259
|
+
ai_tool: { S: isAi ? dev.tool : 'none' },
|
|
260
|
+
ai_model: { S: isAi ? dev.model : '' },
|
|
261
|
+
origin_source: { S: 'write-time-join' },
|
|
262
|
+
// Sparse by-user GSI keys — the per-developer productivity query
|
|
263
|
+
// reads gsi_user / gsi_user_sk rather than scanning.
|
|
264
|
+
gsi_user: { S: `USER#${dev.email}` },
|
|
265
|
+
gsi_user_sk: { S: `COMMIT#${timestamp}` },
|
|
266
|
+
demo_seed: { BOOL: true },
|
|
267
|
+
seed_run_id: { S: seedRunId },
|
|
268
|
+
},
|
|
269
|
+
},
|
|
270
|
+
});
|
|
271
|
+
if (!isAi)
|
|
272
|
+
return;
|
|
273
|
+
const inputTokens = randomInt(3000, 18000);
|
|
274
|
+
const outputTokens = randomInt(800, 6000);
|
|
275
|
+
const costUsd = Math.round((inputTokens / 1000 * 0.003 + outputTokens / 1000 * 0.015) * 10000) / 10000;
|
|
276
|
+
spanRequests.push({
|
|
277
|
+
PutRequest: {
|
|
278
|
+
Item: {
|
|
279
|
+
pk: { S: `USER#${dev.email}` },
|
|
280
|
+
sk: { S: `SPAN#${timestamp}#${hex(8)}` },
|
|
281
|
+
record_type: { S: 'OTEL_SPAN' },
|
|
282
|
+
trace_id: { S: traceId },
|
|
283
|
+
tool: { S: dev.tool },
|
|
284
|
+
model: { S: dev.model },
|
|
285
|
+
input_tokens: { N: String(inputTokens) },
|
|
286
|
+
output_tokens: { N: String(outputTokens) },
|
|
287
|
+
cost_usd: { N: String(costUsd) },
|
|
288
|
+
project: { S: repoName },
|
|
289
|
+
device_id: { S: `demo-${seedRunId}` },
|
|
290
|
+
cost_estimated: { BOOL: false },
|
|
291
|
+
timestamp: { S: timestamp },
|
|
292
|
+
ttl: { N: String(spanTtl) },
|
|
293
|
+
demo_seed: { BOOL: true },
|
|
294
|
+
seed_run_id: { S: seedRunId },
|
|
295
|
+
},
|
|
296
|
+
},
|
|
297
|
+
});
|
|
298
|
+
const day = timestamp.slice(0, 10);
|
|
299
|
+
const key = `${dev.email}|${day}|${dev.tool}|${dev.model}`;
|
|
300
|
+
const roll = dayRollups.get(key) ?? {
|
|
301
|
+
user: dev.email, day, tool: dev.tool, model: dev.model,
|
|
302
|
+
inputTokens: 0, outputTokens: 0, costUsd: 0, calls: 0,
|
|
303
|
+
};
|
|
304
|
+
roll.inputTokens += inputTokens;
|
|
305
|
+
roll.outputTokens += outputTokens;
|
|
306
|
+
roll.costUsd = Math.round((roll.costUsd + costUsd) * 10000) / 10000;
|
|
307
|
+
roll.calls += 1;
|
|
308
|
+
dayRollups.set(key, roll);
|
|
309
|
+
}
|
|
83
310
|
for (let day = 7; day >= 0; day--) {
|
|
84
311
|
const date = new Date();
|
|
85
312
|
date.setDate(date.getDate() - day);
|
|
86
313
|
const dateStr = formatDate(date);
|
|
87
314
|
const maxH = day === 0 ? Math.max(0, new Date().getUTCHours() - 1) : 17;
|
|
88
315
|
const minH = Math.min(8, maxH);
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
for (let i = 0; i < commits; i++) {
|
|
92
|
-
const idx = randomInt(0, 4);
|
|
93
|
-
const origin = ORIGINS[idx];
|
|
94
|
-
const tool = TOOLS[idx];
|
|
95
|
-
const h = randomInt(minH, maxH);
|
|
96
|
-
const m = randomInt(0, 59);
|
|
97
|
-
const ts = `${dateStr}T${String(h).padStart(2, '0')}:${String(m).padStart(2, '0')}:00Z`;
|
|
98
|
-
const lines = randomInt(10, 129);
|
|
99
|
-
addEvent('prism.d1.commit', {
|
|
100
|
-
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
101
|
-
metric: { name: 'commit', value: 1, unit: 'count' },
|
|
102
|
-
ai_context: { tool, model: 'us.anthropic.claude-haiku-4-5-20251001-v1:0', origin },
|
|
103
|
-
ai_dora: { ai_to_merge_ratio: origin !== 'human' ? 1 : 0 },
|
|
104
|
-
});
|
|
105
|
-
}
|
|
106
|
-
// 2-4 PRs per day
|
|
316
|
+
const at = (h, m) => `${dateStr}T${String(h).padStart(2, '0')}:${String(m).padStart(2, '0')}:00Z`;
|
|
317
|
+
// ---- Merged PRs: the census half of Attribution Coverage ----
|
|
107
318
|
const prs = randomInt(2, 4);
|
|
108
319
|
for (let i = 0; i < prs; i++) {
|
|
320
|
+
const prRepo = REPOS[randomInt(0, REPOS.length - 1)];
|
|
321
|
+
const dev = DEVELOPERS[randomInt(0, DEVELOPERS.length - 1)];
|
|
109
322
|
const tc = randomInt(2, 6);
|
|
110
|
-
const
|
|
111
|
-
const
|
|
323
|
+
const shas = Array.from({ length: tc }, () => hex(20));
|
|
324
|
+
const isFailureFix = randomInt(0, 7) === 0;
|
|
112
325
|
const lt = 3600 * (day + 1) + randomInt(0, 3599);
|
|
113
|
-
const
|
|
114
|
-
const
|
|
115
|
-
|
|
116
|
-
//
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
326
|
+
const ts = at(randomInt(minH, maxH), 30);
|
|
327
|
+
const prNumber = 100 + day * 10 + i;
|
|
328
|
+
// Shape mirrors bootstrapper/github-workflows/prism-ai-metrics.yml.
|
|
329
|
+
// Three fields were previously missing or misplaced, each with a
|
|
330
|
+
// visible consequence: pr.total_commits (coverage read "no CI commit
|
|
331
|
+
// census"), pr.commit_shas (security findings had nothing to join for
|
|
332
|
+
// origin), and dora.is_failure_fix (Revert Rate read 0% and Revert
|
|
333
|
+
// Turnaround read "—"). dora.deployment_frequency is deliberately NOT
|
|
334
|
+
// here: it lives only on the deploy event, and carrying it on both
|
|
335
|
+
// double-counted DeploymentFrequency, inflating Merge Frequency 2x.
|
|
120
336
|
addEvent('prism.d1.pr', {
|
|
121
|
-
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
122
|
-
metric: { name: '
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
337
|
+
team_id: team, repo: prRepo, timestamp: ts, prism_level: 2,
|
|
338
|
+
metric: { name: 'pr_merged', value: 1, unit: 'count' },
|
|
339
|
+
dora: { lead_time_seconds: lt, is_failure_fix: isFailureFix },
|
|
340
|
+
pr: {
|
|
341
|
+
number: prNumber,
|
|
342
|
+
author: dev.email.split('@')[0],
|
|
343
|
+
reviews_approved: randomInt(1, 2),
|
|
344
|
+
reviews_changes_requested: randomInt(0, 1),
|
|
127
345
|
total_commits: tc,
|
|
128
|
-
|
|
129
|
-
total_input_tokens: inputTokens,
|
|
130
|
-
total_output_tokens: outputTokens,
|
|
131
|
-
total_cost_usd: costUsd,
|
|
346
|
+
commit_shas: shas,
|
|
132
347
|
},
|
|
133
|
-
pr: { number: pr, author: 'engineer' },
|
|
134
348
|
});
|
|
135
|
-
// Deploy event
|
|
136
349
|
addEvent('prism.d1.deploy', {
|
|
137
|
-
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
350
|
+
team_id: team, repo: prRepo, timestamp: ts, prism_level: 2,
|
|
138
351
|
metric: { name: 'deployment', value: 1, unit: 'count' },
|
|
139
352
|
dora: { deployment_frequency: 1 },
|
|
140
353
|
});
|
|
354
|
+
// Commit events and attribution facts describe the SAME commits, so the
|
|
355
|
+
// two stores agree. Emitting them independently is how the previous
|
|
356
|
+
// version drifted.
|
|
357
|
+
for (const sha of shas) {
|
|
358
|
+
const cts = at(randomInt(minH, maxH), randomInt(0, 59));
|
|
359
|
+
const isAi = dev.onboarded && Math.random() < dev.aiRate;
|
|
360
|
+
const inMain = randomInt(0, 19) !== 0; // ~95% merged
|
|
361
|
+
const wasReverted = inMain && randomInt(0, 24) === 0; // ~4% of merged
|
|
362
|
+
addEvent('prism.d1.commit', {
|
|
363
|
+
team_id: team, repo: prRepo, timestamp: cts, prism_level: 2,
|
|
364
|
+
metric: { name: 'commit', value: 1, unit: 'count' },
|
|
365
|
+
ai_context: {
|
|
366
|
+
tool: isAi ? dev.tool : 'n/a',
|
|
367
|
+
model: isAi ? dev.model : 'n/a',
|
|
368
|
+
origin: isAi ? 'ai-generated' : 'human',
|
|
369
|
+
},
|
|
370
|
+
});
|
|
371
|
+
if (seedAttribution && dev.onboarded) {
|
|
372
|
+
seedCommit(dev, prRepo, sha, cts, isAi, inMain, wasReverted);
|
|
373
|
+
}
|
|
374
|
+
}
|
|
141
375
|
}
|
|
142
|
-
// Eval results
|
|
376
|
+
// ---- Eval gate results ----
|
|
143
377
|
for (let i = 0; i < prs; i++) {
|
|
144
378
|
let score = 0.88;
|
|
145
379
|
let res = 'PASS';
|
|
@@ -151,23 +385,24 @@ export default {
|
|
|
151
385
|
score = 0.71;
|
|
152
386
|
res = 'FAIL';
|
|
153
387
|
}
|
|
154
|
-
const ts = `${dateStr}T${String(randomInt(minH, maxH)).padStart(2, '0')}:45:00Z`;
|
|
155
388
|
addEvent('prism.d1.eval', {
|
|
156
|
-
team_id: team, repo, timestamp:
|
|
389
|
+
team_id: team, repo, timestamp: at(randomInt(minH, maxH), 45), prism_level: 2,
|
|
157
390
|
metric: { name: 'eval_score', value: score, unit: 'score' },
|
|
158
|
-
ai_context: { tool: 'bedrock-eval', model:
|
|
391
|
+
ai_context: { tool: 'bedrock-eval', model: HAIKU, origin: 'ai-generated' },
|
|
159
392
|
ai_dora: { eval_gate_pass_rate: res === 'PASS' ? 1 : 0 },
|
|
160
393
|
eval: { result: res, pr_number: 100 + day * 10 + i },
|
|
161
394
|
});
|
|
162
395
|
}
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
396
|
+
const ts = at(9, 0);
|
|
397
|
+
// Change failure rate and MTTR still have live CloudWatch metrics and an
|
|
398
|
+
// alarm. The other assessment metrics this loop used to emit —
|
|
399
|
+
// ai_acceptance_rate, ai_test_coverage_delta, spec_to_code_hours,
|
|
400
|
+
// post_merge_defect_rate, PRISMLevel, BedrockCostUSD, TokenEfficiency —
|
|
401
|
+
// have no consumer left: their metrics were deleted, their producing
|
|
402
|
+
// Lambdas were removed, or the exec view now computes the value rather
|
|
403
|
+
// than reading a published one.
|
|
404
|
+
const cfr = parseFloat((randomInt(1, 8) / 100).toFixed(4));
|
|
405
|
+
const mttrSeconds = randomInt(600, 4199);
|
|
171
406
|
addEvent('prism.d1.assessment', {
|
|
172
407
|
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
173
408
|
metric: { name: 'change_failure_rate', value: cfr, unit: 'percent' },
|
|
@@ -175,122 +410,73 @@ export default {
|
|
|
175
410
|
});
|
|
176
411
|
addEvent('prism.d1.assessment', {
|
|
177
412
|
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
178
|
-
metric: { name: 'mttr', value:
|
|
179
|
-
dora: { mttr_seconds:
|
|
413
|
+
metric: { name: 'mttr', value: mttrSeconds, unit: 'seconds' },
|
|
414
|
+
dora: { mttr_seconds: mttrSeconds },
|
|
180
415
|
});
|
|
181
|
-
|
|
182
|
-
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
183
|
-
metric: { name: 'ai_acceptance_rate', value: accept, unit: 'percent' },
|
|
184
|
-
ai_dora: { ai_acceptance_rate: accept },
|
|
185
|
-
});
|
|
186
|
-
addEvent('prism.d1.assessment', {
|
|
187
|
-
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
188
|
-
metric: { name: 'ai_test_coverage_delta', value: coverage, unit: 'percent' },
|
|
189
|
-
ai_dora: { ai_test_coverage_delta: coverage },
|
|
190
|
-
});
|
|
191
|
-
addEvent('prism.d1.assessment', {
|
|
192
|
-
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
193
|
-
metric: { name: 'spec_to_code_hours', value: specH, unit: 'count' },
|
|
194
|
-
ai_dora: { spec_to_code_hours: specH },
|
|
195
|
-
});
|
|
196
|
-
addEvent('prism.d1.assessment', {
|
|
197
|
-
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
198
|
-
metric: { name: 'post_merge_defect_rate', value: defect, unit: 'percent' },
|
|
199
|
-
ai_dora: { post_merge_defect_rate: defect },
|
|
200
|
-
});
|
|
201
|
-
// PRISM Level (trending up over the week)
|
|
202
|
-
const level = parseFloat((1.5 + (7 - day) * 0.2).toFixed(1));
|
|
203
|
-
addEvent('prism.d1.assessment', {
|
|
204
|
-
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
205
|
-
metric: { name: 'PRISMLevel', value: level, unit: 'none' },
|
|
206
|
-
});
|
|
207
|
-
// Agent invocations
|
|
416
|
+
// ---- Agent invocations ----
|
|
208
417
|
const agentCount = randomInt(3, 10);
|
|
209
418
|
for (let a = 0; a < agentCount; a++) {
|
|
210
419
|
const steps = randomInt(2, 9);
|
|
211
|
-
const dur = randomInt(1000, 5999);
|
|
212
|
-
const tokens = randomInt(2000, 9999);
|
|
213
|
-
const status = randomInt(0, 5) === 0 ? 'failure' : 'success';
|
|
214
|
-
const ats = `${dateStr}T${String(randomInt(minH, maxH)).padStart(2, '0')}:${String(randomInt(0, 59)).padStart(2, '0')}:00Z`;
|
|
215
420
|
addEvent('prism.d1.agent', {
|
|
216
|
-
team_id: team, repo, timestamp:
|
|
421
|
+
team_id: team, repo, timestamp: at(randomInt(minH, maxH), randomInt(0, 59)), prism_level: 3,
|
|
217
422
|
metric: { name: 'agent_invocation', value: 1, unit: 'count' },
|
|
218
|
-
ai_context: { tool: 'strands-agent', model:
|
|
423
|
+
ai_context: { tool: 'strands-agent', model: HAIKU, origin: 'ai-generated' },
|
|
219
424
|
agent: {
|
|
220
425
|
agent_name: 'task-assistant', steps_taken: steps, tools_invoked: steps - 1,
|
|
221
|
-
duration_ms:
|
|
426
|
+
duration_ms: randomInt(1000, 5999), tokens_used: randomInt(2000, 9999),
|
|
427
|
+
status: randomInt(0, 5) === 0 ? 'failure' : 'success', guardrails_triggered: 0,
|
|
222
428
|
},
|
|
223
429
|
});
|
|
224
430
|
}
|
|
225
|
-
// Guardrail triggers
|
|
431
|
+
// ---- Guardrail triggers ----
|
|
226
432
|
const GUARDRAIL_CATEGORIES = ['CONTENT_FILTER', 'DENIED_TOPIC', 'SENSITIVE_INFO', 'WORD_FILTER'];
|
|
227
433
|
const GUARDRAIL_ACTIONS = ['BLOCK', 'ANONYMIZE', 'WARN'];
|
|
228
|
-
|
|
229
|
-
for (let g = 0; g < guardrailCount; g++) {
|
|
230
|
-
const gts = `${dateStr}T${String(randomInt(minH, maxH)).padStart(2, '0')}:${String(randomInt(0, 59)).padStart(2, '0')}:00Z`;
|
|
231
|
-
const category = GUARDRAIL_CATEGORIES[randomInt(0, 3)];
|
|
232
|
-
const action = GUARDRAIL_ACTIONS[randomInt(0, 2)];
|
|
434
|
+
for (let g = 0; g < randomInt(1, 5); g++) {
|
|
233
435
|
addEvent('prism.d1.guardrail', {
|
|
234
|
-
team_id: team, repo, timestamp:
|
|
436
|
+
team_id: team, repo, timestamp: at(randomInt(minH, maxH), randomInt(0, 59)), prism_level: 2,
|
|
235
437
|
metric: { name: 'guardrail_trigger', value: 1, unit: 'count' },
|
|
236
438
|
guardrail: {
|
|
237
439
|
guardrail_id: 'gr-demo-001', guardrail_name: 'prism-safety',
|
|
238
|
-
trigger_category:
|
|
239
|
-
action_taken:
|
|
440
|
+
trigger_category: GUARDRAIL_CATEGORIES[randomInt(0, 3)], trigger_type: 'automated',
|
|
441
|
+
action_taken: GUARDRAIL_ACTIONS[randomInt(0, 2)], agent_name: 'task-assistant',
|
|
442
|
+
invocation_id: `inv-${day}-${g}`,
|
|
240
443
|
},
|
|
241
444
|
});
|
|
242
445
|
}
|
|
243
|
-
// MCP tool calls
|
|
446
|
+
// ---- MCP tool calls ----
|
|
244
447
|
const MCP_TOOLS = ['file_read', 'file_write', 'shell_exec', 'web_fetch', 'db_query'];
|
|
245
|
-
|
|
246
|
-
for (let m = 0; m < mcpCount; m++) {
|
|
247
|
-
const mts = `${dateStr}T${String(randomInt(minH, maxH)).padStart(2, '0')}:${String(randomInt(0, 59)).padStart(2, '0')}:00Z`;
|
|
448
|
+
for (let m = 0; m < randomInt(5, 15); m++) {
|
|
248
449
|
const authorized = randomInt(0, 19) !== 0; // 5% denied
|
|
249
450
|
addEvent('prism.d1.mcp.tool_call', {
|
|
250
|
-
team_id: team, repo, timestamp:
|
|
451
|
+
team_id: team, repo, timestamp: at(randomInt(minH, maxH), randomInt(0, 59)), prism_level: 2,
|
|
251
452
|
metric: { name: 'mcp_tool_call', value: 1, unit: 'count' },
|
|
252
453
|
mcp_tool_call: {
|
|
253
454
|
session_id: `sess-${day}-${m}`, client_id: 'claude-code',
|
|
254
|
-
tool_name: MCP_TOOLS[randomInt(0, 4)],
|
|
255
|
-
scopes_used: ['read'], authorized,
|
|
455
|
+
tool_name: MCP_TOOLS[randomInt(0, 4)], scopes_used: ['read'], authorized,
|
|
256
456
|
risk_level: authorized ? 'low' : 'high',
|
|
257
457
|
duration_ms: randomInt(50, 2000),
|
|
258
458
|
result_status: authorized ? 'success' : 'denied',
|
|
259
459
|
},
|
|
260
460
|
});
|
|
261
461
|
}
|
|
262
|
-
//
|
|
263
|
-
const dailyCost = parseFloat((randomInt(15, 45) + randomInt(0, 99) / 100).toFixed(2));
|
|
264
|
-
const tokenEff = randomInt(80, 300);
|
|
265
|
-
addEvent('prism.d1.assessment', {
|
|
266
|
-
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
267
|
-
metric: { name: 'BedrockCostUSD', value: dailyCost, unit: 'none' },
|
|
268
|
-
});
|
|
269
|
-
addEvent('prism.d1.assessment', {
|
|
270
|
-
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
271
|
-
metric: { name: 'TokenEfficiency', value: tokenEff, unit: 'none' },
|
|
272
|
-
});
|
|
273
|
-
// Quality: AI vs Human defect rates (Pillar 7)
|
|
274
|
-
const aiDefect = parseFloat((randomInt(1, 4) / 100).toFixed(4));
|
|
275
|
-
const humanDefect = parseFloat((randomInt(2, 7) / 100).toFixed(4));
|
|
462
|
+
// ---- Quality: AI vs human defect rates ----
|
|
276
463
|
addEvent('prism.d1.quality', {
|
|
277
464
|
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
278
465
|
metric: { name: 'quality_comparison', value: 1, unit: 'count' },
|
|
279
466
|
quality: {
|
|
280
467
|
deployment_id: `deploy-${day}`,
|
|
281
|
-
ai_defect_rate:
|
|
468
|
+
ai_defect_rate: parseFloat((randomInt(1, 4) / 100).toFixed(4)),
|
|
469
|
+
human_defect_rate: parseFloat((randomInt(2, 7) / 100).toFixed(4)),
|
|
282
470
|
total_ai_commits: randomInt(5, 12), total_human_commits: randomInt(3, 8),
|
|
283
471
|
},
|
|
284
472
|
});
|
|
285
|
-
// Security
|
|
473
|
+
// ---- Security findings ----
|
|
286
474
|
const PHASES = ['design_review', 'code_review', 'pen_test'];
|
|
287
475
|
const SEVERITIES = ['CRITICAL', 'HIGH', 'MEDIUM', 'LOW'];
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
const fts = `${dateStr}T${String(randomInt(minH, maxH)).padStart(2, '0')}:${String(randomInt(0, 59)).padStart(2, '0')}:00Z`;
|
|
476
|
+
for (let f = 0; f < randomInt(1, 4); f++) {
|
|
477
|
+
const fts = at(randomInt(minH, maxH), randomInt(0, 59));
|
|
291
478
|
const severity = SEVERITIES[randomInt(0, 3)];
|
|
292
479
|
const phase = PHASES[randomInt(0, 2)];
|
|
293
|
-
const aiOrigin = randomInt(0, 1) === 0 ? 'ai-assisted' : 'human';
|
|
294
480
|
addEvent('prism.d1.security.code_review', {
|
|
295
481
|
team_id: team, repo, timestamp: fts, prism_level: 2,
|
|
296
482
|
metric: { name: 'security_finding', value: 1, unit: 'count' },
|
|
@@ -299,60 +485,98 @@ export default {
|
|
|
299
485
|
cvss_score: severity === 'CRITICAL' ? 9.1 : severity === 'HIGH' ? 7.5 : severity === 'MEDIUM' ? 5.2 : 2.8,
|
|
300
486
|
title: 'Demo finding', category: 'injection',
|
|
301
487
|
cwe_id: 'CWE-79', exploit_validated: phase === 'pen_test',
|
|
302
|
-
compliance_mappings: ['OWASP-A03'],
|
|
488
|
+
compliance_mappings: ['OWASP-A03'],
|
|
489
|
+
ai_origin: randomInt(0, 1) === 0 ? 'ai-assisted' : 'human',
|
|
303
490
|
spec_ref: null, found_at: fts, remediated_at: null,
|
|
304
491
|
},
|
|
305
492
|
});
|
|
306
493
|
}
|
|
307
|
-
// Security remediation
|
|
494
|
+
// ---- Security remediation ----
|
|
308
495
|
if (day > 0 && randomInt(0, 2) === 0) {
|
|
309
|
-
const rts = `${dateStr}T${String(randomInt(minH, maxH)).padStart(2, '0')}:${String(randomInt(0, 59)).padStart(2, '0')}:00Z`;
|
|
310
|
-
const remOrigin = randomInt(0, 1) === 0 ? 'ai-assisted' : 'human';
|
|
311
496
|
addEvent('prism.d1.security.remediation', {
|
|
312
|
-
team_id: team, repo, timestamp:
|
|
497
|
+
team_id: team, repo, timestamp: at(randomInt(minH, maxH), randomInt(0, 59)), prism_level: 2,
|
|
313
498
|
metric: { name: 'security_remediation', value: 1, unit: 'count' },
|
|
314
499
|
security_remediation: {
|
|
315
500
|
finding_id: `finding-${day + 1}-0`,
|
|
316
501
|
severity: SEVERITIES[randomInt(0, 2)],
|
|
317
502
|
remediation_time_hours: randomInt(2, 48),
|
|
318
|
-
remediated_by_origin:
|
|
503
|
+
remediated_by_origin: randomInt(0, 1) === 0 ? 'ai-assisted' : 'human',
|
|
319
504
|
finding_phase: 'code_review',
|
|
320
505
|
},
|
|
321
506
|
});
|
|
322
507
|
}
|
|
323
|
-
// Exfiltration alert (rare)
|
|
508
|
+
// ---- Exfiltration alert (rare) ----
|
|
324
509
|
if (randomInt(0, 6) === 0) {
|
|
325
|
-
const ets =
|
|
510
|
+
const ets = at(randomInt(minH, maxH), randomInt(0, 59));
|
|
326
511
|
addEvent('prism.d1.security', {
|
|
327
512
|
team_id: team, repo, timestamp: ets, prism_level: 2,
|
|
328
513
|
metric: { name: 'exfiltration_alert', value: 1, unit: 'count' },
|
|
329
514
|
security: {
|
|
330
515
|
alert_type: 'anomalous_read', table_name: 'prism-d1-events',
|
|
331
516
|
principal_arn: 'arn:aws:iam::123456789012:role/demo-role',
|
|
332
|
-
read_count: randomInt(500, 2000),
|
|
333
|
-
window_start: ets, window_end: ets,
|
|
517
|
+
read_count: randomInt(500, 2000), window_start: ets, window_end: ets,
|
|
334
518
|
},
|
|
335
519
|
});
|
|
336
520
|
}
|
|
337
|
-
//
|
|
521
|
+
// ---- Per-rubric eval scores ----
|
|
338
522
|
const RUBRICS = ['code-quality', 'api-response-quality', 'agent-quality', 'security-compliance', 'spec-compliance'];
|
|
339
523
|
for (const rubric of RUBRICS) {
|
|
340
524
|
const rScore = parseFloat((randomInt(65, 98) / 100).toFixed(2));
|
|
341
|
-
const rts = `${dateStr}T${String(randomInt(minH, maxH)).padStart(2, '0')}:50:00Z`;
|
|
342
525
|
addEvent('prism.d1.eval', {
|
|
343
|
-
team_id: team, repo, timestamp:
|
|
526
|
+
team_id: team, repo, timestamp: at(randomInt(minH, maxH), 50), prism_level: 2,
|
|
344
527
|
metric: { name: 'eval_score', value: rScore, unit: 'score' },
|
|
345
|
-
ai_context: { tool: 'bedrock-eval', model:
|
|
528
|
+
ai_context: { tool: 'bedrock-eval', model: HAIKU, origin: 'ai-generated' },
|
|
346
529
|
ai_dora: { eval_gate_pass_rate: rScore >= 0.7 ? 1 : 0 },
|
|
347
530
|
eval: { result: rScore >= 0.7 ? 'PASS' : 'FAIL', rubric, score: rScore, pr_number: 100 + day * 10 },
|
|
348
531
|
});
|
|
349
532
|
}
|
|
350
533
|
}
|
|
351
534
|
flush();
|
|
352
|
-
console.log('
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
535
|
+
console.log('');
|
|
536
|
+
// ---- Attribution store ----
|
|
537
|
+
let attrWritten = 0;
|
|
538
|
+
if (seedAttribution) {
|
|
539
|
+
for (const roll of dayRollups.values()) {
|
|
540
|
+
spanRequests.push({
|
|
541
|
+
PutRequest: {
|
|
542
|
+
Item: {
|
|
543
|
+
pk: { S: `USER#${roll.user}` },
|
|
544
|
+
// '#' is the key delimiter, so it must not appear in the model.
|
|
545
|
+
sk: { S: `OTEL#DAY#${roll.day}#${roll.tool}#${roll.model.replace(/#/g, '')}` },
|
|
546
|
+
record_type: { S: 'OTEL_DAY' },
|
|
547
|
+
tool: { S: roll.tool },
|
|
548
|
+
model: { S: roll.model.replace(/#/g, '') },
|
|
549
|
+
day: { S: roll.day },
|
|
550
|
+
input_tokens: { N: String(roll.inputTokens) },
|
|
551
|
+
output_tokens: { N: String(roll.outputTokens) },
|
|
552
|
+
cost_usd: { N: String(roll.costUsd) },
|
|
553
|
+
call_count: { N: String(roll.calls) },
|
|
554
|
+
updated_at: { S: new Date().toISOString() },
|
|
555
|
+
demo_seed: { BOOL: true },
|
|
556
|
+
seed_run_id: { S: seedRunId },
|
|
557
|
+
},
|
|
558
|
+
},
|
|
559
|
+
});
|
|
560
|
+
}
|
|
561
|
+
const totalItems = spanRequests.length + commitRequests.length;
|
|
562
|
+
if (totalItems > 0) {
|
|
563
|
+
console.log(`Seeding ${totalItems} attribution items into ${table}...`);
|
|
564
|
+
// Spans and rollups first — see the note on the accumulators above.
|
|
565
|
+
attrWritten += ddbBatchWrite(region, table, spanRequests);
|
|
566
|
+
attrWritten += ddbBatchWrite(region, table, commitRequests);
|
|
567
|
+
console.log('');
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
console.log(`=== Done! ${total} events emitted, ${attrWritten} attribution items seeded ===`);
|
|
571
|
+
if (seedAttribution) {
|
|
572
|
+
const onboarded = DEVELOPERS.filter(d => d.onboarded).length;
|
|
573
|
+
console.log(`Fleet: ${onboarded} of ${DEVELOPERS.length} developers onboarded — Attribution Coverage`);
|
|
574
|
+
console.log('will read below 100% by design, which is what the KPI is for.');
|
|
575
|
+
console.log(`Seed run ${seedRunId} · remove with: prism-cli workshop generate-demo-data --purge-demo`);
|
|
576
|
+
}
|
|
577
|
+
console.log('');
|
|
578
|
+
console.log(`Open CloudWatch → Dashboards → PRISM-D1-Team-Velocity (${region})`);
|
|
579
|
+
console.log("Set the time range to 'Last 1 week'.");
|
|
356
580
|
},
|
|
357
581
|
};
|
|
358
582
|
//# sourceMappingURL=generate-demo-data.js.map
|