@prism-d1/cli 1.0.80 → 1.0.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/github-workflows/prism-ai-metrics.yml +1 -1
- package/dist/assets/gitlab-workflows/prism-ai-metrics.yml +1 -1
- package/dist/assets/infra/lib/metrics-pipeline-stack.ts +3 -2
- package/dist/src/commands/assessment/web.d.ts.map +1 -1
- package/dist/src/commands/assessment/web.js +25 -2
- package/dist/src/commands/assessment/web.js.map +1 -1
- package/dist/src/commands/bootstrapper/install-eval-harness.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/install-eval-harness.js +15 -5
- package/dist/src/commands/bootstrapper/install-eval-harness.js.map +1 -1
- package/dist/src/commands/bootstrapper/install-git-hooks.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/install-git-hooks.js +10 -14
- package/dist/src/commands/bootstrapper/install-git-hooks.js.map +1 -1
- package/dist/src/commands/bootstrapper/setup-github-oidc.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/setup-github-oidc.js +50 -27
- package/dist/src/commands/bootstrapper/setup-github-oidc.js.map +1 -1
- package/dist/src/commands/bootstrapper/setup-gitlab-oidc.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/setup-gitlab-oidc.js +69 -25
- package/dist/src/commands/bootstrapper/setup-gitlab-oidc.js.map +1 -1
- package/dist/src/commands/bootstrapper/setup-otel-sync.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/setup-otel-sync.js +145 -53
- package/dist/src/commands/bootstrapper/setup-otel-sync.js.map +1 -1
- package/dist/src/commands/workshop/generate-demo-data.d.ts +41 -2
- package/dist/src/commands/workshop/generate-demo-data.d.ts.map +1 -1
- package/dist/src/commands/workshop/generate-demo-data.js +502 -166
- package/dist/src/commands/workshop/generate-demo-data.js.map +1 -1
- package/dist/src/commands/workshop/verify-setup.d.ts.map +1 -1
- package/dist/src/commands/workshop/verify-setup.js +75 -14
- package/dist/src/commands/workshop/verify-setup.js.map +1 -1
- package/dist/src/utils/exec.d.ts +56 -0
- package/dist/src/utils/exec.d.ts.map +1 -0
- package/dist/src/utils/exec.js +72 -0
- package/dist/src/utils/exec.js.map +1 -0
- package/dist/src/utils/tempfile.d.ts +19 -0
- package/dist/src/utils/tempfile.d.ts.map +1 -0
- package/dist/src/utils/tempfile.js +32 -0
- package/dist/src/utils/tempfile.js.map +1 -0
- package/dist/src/utils/validate.d.ts +54 -0
- package/dist/src/utils/validate.d.ts.map +1 -0
- package/dist/src/utils/validate.js +143 -0
- package/dist/src/utils/validate.js.map +1 -0
- package/package.json +1 -1
|
@@ -1,60 +1,287 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
/**
|
|
2
|
+
* PRISM D1 — workshop demo data generator.
|
|
3
|
+
*
|
|
4
|
+
* The dashboards read TWO stores, and a generator that fills only one leaves
|
|
5
|
+
* roughly half of every view blank:
|
|
6
|
+
*
|
|
7
|
+
* EventBridge → metrics-processor → events table + CloudWatch
|
|
8
|
+
* Delivery KPIs, eval gates, governance, agent ops, security, most of CISO.
|
|
9
|
+
*
|
|
10
|
+
* AI-usage table (attribution) → otel-receiver GET routes
|
|
11
|
+
* AI share, merge rate, defect rate, $/shipped commit, Attribution
|
|
12
|
+
* Coverage, Repository Breakdown, the observed PRISM level, and the whole
|
|
13
|
+
* Developer Productivity dashboard.
|
|
14
|
+
*
|
|
15
|
+
* Attribution is written directly to DynamoDB rather than POSTed to
|
|
16
|
+
* /v1/traces because the receiver derives the owning developer from the
|
|
17
|
+
* caller's JWT claim. Every seeded commit would land under the single
|
|
18
|
+
* participant running this command, and a one-developer fleet cannot
|
|
19
|
+
* demonstrate the per-developer view or a coverage shortfall. Direct writes
|
|
20
|
+
* are the only way to seed a multi-developer fleet.
|
|
21
|
+
*
|
|
22
|
+
* The cost of that choice is schema coupling. Item shapes below must match
|
|
23
|
+
* infra/lib/lambda/otel-receiver.ts — specifically writeCommitAttribution,
|
|
24
|
+
* writeSpanIfNew and bumpDailyAggregate. If those change, this changes too.
|
|
25
|
+
*
|
|
26
|
+
* Guardrails, because this writes fabricated facts into a metrics store:
|
|
27
|
+
* - every seeded item carries demo_seed=true and a seed_run_id
|
|
28
|
+
* - seeding refuses to run when the table already holds real attribution,
|
|
29
|
+
* unless --force is passed
|
|
30
|
+
* - --purge-demo removes seeded items and leaves real ones alone
|
|
31
|
+
*/
|
|
32
|
+
import { randomBytes } from 'node:crypto';
|
|
33
|
+
import { run } from '../../utils/exec.js';
|
|
34
|
+
import { withPrivateTemp } from '../../utils/tempfile.js';
|
|
35
|
+
// ---- Shell-free AWS CLI helpers ----
|
|
36
|
+
/**
|
|
37
|
+
* EventBridge put-events with the entry array passed as a file.
|
|
38
|
+
*
|
|
39
|
+
* The previous form interpolated JSON.stringify output into a single-quoted
|
|
40
|
+
* shell string (`--entries '${json}'`). A team, bus or repo name containing an
|
|
41
|
+
* apostrophe terminated that quoted region and the remainder became shell
|
|
42
|
+
* source. Passing a file path removes both the quoting and the shell.
|
|
43
|
+
*/
|
|
44
|
+
function putEvents(region, entries) {
|
|
45
|
+
return withPrivateTemp('entries.json', JSON.stringify(entries), (file) => run('aws', ['events', 'put-events', '--region', region, '--entries', `file://${file}`]).ok);
|
|
46
|
+
}
|
|
47
|
+
/** Write one request with the single-item APIs (PutItem / DeleteItem). */
|
|
48
|
+
function ddbWriteOne(region, table, req) {
|
|
49
|
+
if ('PutRequest' in req) {
|
|
50
|
+
const r = withPrivateTemp('item.json', JSON.stringify(req.PutRequest.Item), (file) => run('aws', ['dynamodb', 'put-item', '--region', region, '--table-name', table, '--item', `file://${file}`]));
|
|
51
|
+
return { ok: r.ok, stderr: r.stderr };
|
|
6
52
|
}
|
|
7
|
-
|
|
8
|
-
|
|
53
|
+
const r = withPrivateTemp('key.json', JSON.stringify(req.DeleteRequest.Key), (file) => run('aws', ['dynamodb', 'delete-item', '--region', region, '--table-name', table, '--key', `file://${file}`]));
|
|
54
|
+
return { ok: r.ok, stderr: r.stderr };
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Whether this caller may use BatchWriteItem. null until the first attempt.
|
|
58
|
+
*
|
|
59
|
+
* Module-scoped so the discovery is made once. When it lives inside
|
|
60
|
+
* ddbBatchWrite it resets per call, and a role without the action pays for a
|
|
61
|
+
* doomed batch request on every invocation — the preflight probe alone
|
|
62
|
+
* re-discovered it three times.
|
|
63
|
+
*/
|
|
64
|
+
let batchWriteSupported = null;
|
|
65
|
+
/**
|
|
66
|
+
* Write requests to DynamoDB, preferring BatchWriteItem and falling back to the
|
|
67
|
+
* single-item APIs.
|
|
68
|
+
*
|
|
69
|
+
* The fallback exists because IAM policies routinely enumerate DynamoDB actions
|
|
70
|
+
* individually. A live workshop role held dynamodb:PutItem — a single put-item
|
|
71
|
+
* succeeded against the real table, KMS included — while every BatchWriteItem
|
|
72
|
+
* call failed. Rather than require an IAM change to a role this repo does not
|
|
73
|
+
* define, fall back to the API that is demonstrably permitted.
|
|
74
|
+
*
|
|
75
|
+
* Per-item writes are slower but bounded: a full seed is roughly 150 items.
|
|
76
|
+
*/
|
|
77
|
+
function ddbBatchWrite(region, table, requests) {
|
|
78
|
+
let written = 0;
|
|
79
|
+
let batchError = '';
|
|
80
|
+
for (let i = 0; i < requests.length; i += 25) {
|
|
81
|
+
const chunk = requests.slice(i, i + 25);
|
|
82
|
+
if (batchWriteSupported !== false) {
|
|
83
|
+
const payload = JSON.stringify({ [table]: chunk });
|
|
84
|
+
const result = withPrivateTemp('batch.json', payload, (file) => run('aws', ['dynamodb', 'batch-write-item', '--region', region, '--request-items', `file://${file}`]));
|
|
85
|
+
if (result.ok) {
|
|
86
|
+
batchWriteSupported = true;
|
|
87
|
+
written += chunk.length;
|
|
88
|
+
process.stdout.write('.');
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
// Remember why, then write everything item by item from here on.
|
|
92
|
+
batchError = result.stderr || 'batch-write-item failed with no stderr';
|
|
93
|
+
batchWriteSupported = false;
|
|
94
|
+
process.stdout.write('!');
|
|
95
|
+
}
|
|
96
|
+
for (const req of chunk) {
|
|
97
|
+
const one = ddbWriteOne(region, table, req);
|
|
98
|
+
if (!one.ok) {
|
|
99
|
+
return {
|
|
100
|
+
written,
|
|
101
|
+
error: `batch-write-item failed and the single-item fallback also failed.\n` +
|
|
102
|
+
` batch-write-item: ${batchError || '(not attempted this call)'}\n` +
|
|
103
|
+
` put-item/delete-item: ${one.stderr}`,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
written++;
|
|
107
|
+
if (written % 25 === 0)
|
|
108
|
+
process.stdout.write('.');
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return { written, error: null };
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Confirm the attribution table is writable *by the API seeding actually uses*
|
|
115
|
+
* before generating anything. Runs regardless of --force, because --force is
|
|
116
|
+
* about overwriting real data, not about reachability.
|
|
117
|
+
*
|
|
118
|
+
* The probe deliberately uses BatchWriteItem rather than PutItem. IAM policies
|
|
119
|
+
* routinely enumerate DynamoDB actions individually, so a role can hold
|
|
120
|
+
* dynamodb:PutItem and not dynamodb:BatchWriteItem — which is exactly the
|
|
121
|
+
* failure this function was added for. Probing PutItem would have passed and
|
|
122
|
+
* left the real failure to surface 150 items later.
|
|
123
|
+
*
|
|
124
|
+
* describe-table alone is also insufficient: read access does not imply write
|
|
125
|
+
* access, and the table is KMS-encrypted, so a role can hold the DynamoDB
|
|
126
|
+
* actions and still fail on kms:GenerateDataKey.
|
|
127
|
+
*/
|
|
128
|
+
function preflightAttributionTable(region, table) {
|
|
129
|
+
const describe = run('aws', ['dynamodb', 'describe-table', '--region', region, '--table-name', table, '--output', 'json']);
|
|
130
|
+
if (!describe.ok) {
|
|
131
|
+
return `cannot describe ${table}: ${describe.stderr}`;
|
|
132
|
+
}
|
|
133
|
+
const probeKey = { pk: { S: 'REPO#__prism_preflight__' }, sk: { S: 'COMMIT#__prism_preflight__' } };
|
|
134
|
+
const probeItem = { ...probeKey, record_type: { S: 'PREFLIGHT' }, demo_seed: { BOOL: true } };
|
|
135
|
+
const put = ddbBatchWrite(region, table, [{ PutRequest: { Item: probeItem } }]);
|
|
136
|
+
if (put.error) {
|
|
137
|
+
return `cannot batch-write to ${table}: ${put.error}`;
|
|
9
138
|
}
|
|
139
|
+
ddbBatchWrite(region, table, [{ DeleteRequest: { Key: probeKey } }]);
|
|
140
|
+
return null;
|
|
10
141
|
}
|
|
11
|
-
|
|
142
|
+
/** True when the table holds attribution commits that this generator did not create. */
|
|
143
|
+
function hasRealAttribution(region, table) {
|
|
144
|
+
const values = JSON.stringify({ ':rt': { S: 'OTEL_ATTR_COMMIT' } });
|
|
145
|
+
const result = withPrivateTemp('values.json', values, (file) => run('aws', [
|
|
146
|
+
'dynamodb', 'scan',
|
|
147
|
+
'--region', region,
|
|
148
|
+
'--table-name', table,
|
|
149
|
+
'--select', 'COUNT',
|
|
150
|
+
'--filter-expression', 'record_type = :rt AND attribute_not_exists(demo_seed)',
|
|
151
|
+
'--expression-attribute-values', `file://${file}`,
|
|
152
|
+
'--output', 'json',
|
|
153
|
+
]));
|
|
154
|
+
if (!result.ok)
|
|
155
|
+
return false; // table missing or no access — surfaced by the caller's first write
|
|
12
156
|
try {
|
|
13
|
-
return
|
|
157
|
+
return (JSON.parse(result.stdout).Count ?? 0) > 0;
|
|
14
158
|
}
|
|
15
159
|
catch {
|
|
16
|
-
return
|
|
160
|
+
return false;
|
|
17
161
|
}
|
|
18
162
|
}
|
|
19
|
-
function
|
|
163
|
+
function purgeDemoAttribution(region, table) {
|
|
164
|
+
const values = JSON.stringify({ ':t': { BOOL: true } });
|
|
165
|
+
const result = withPrivateTemp('values.json', values, (file) => run('aws', [
|
|
166
|
+
'dynamodb', 'scan',
|
|
167
|
+
'--region', region,
|
|
168
|
+
'--table-name', table,
|
|
169
|
+
'--filter-expression', 'demo_seed = :t',
|
|
170
|
+
'--projection-expression', 'pk,sk',
|
|
171
|
+
'--expression-attribute-values', `file://${file}`,
|
|
172
|
+
'--output', 'json',
|
|
173
|
+
]));
|
|
174
|
+
if (!result.ok) {
|
|
175
|
+
console.error(`Error: could not scan ${table}: ${result.stderr}`);
|
|
176
|
+
return 0;
|
|
177
|
+
}
|
|
178
|
+
let items;
|
|
20
179
|
try {
|
|
21
|
-
|
|
22
|
-
return true;
|
|
180
|
+
items = JSON.parse(result.stdout).Items ?? [];
|
|
23
181
|
}
|
|
24
182
|
catch {
|
|
25
|
-
|
|
183
|
+
console.error('Error: could not parse scan output');
|
|
184
|
+
return 0;
|
|
26
185
|
}
|
|
186
|
+
if (items.length === 0)
|
|
187
|
+
return 0;
|
|
188
|
+
return ddbBatchWrite(region, table, items.map(key => ({ DeleteRequest: { Key: key } }))).written;
|
|
27
189
|
}
|
|
190
|
+
// ---- Synthetic fleet ----
|
|
191
|
+
const SONNET = 'us.anthropic.claude-sonnet-4-5-20250929-v1:0';
|
|
192
|
+
const HAIKU = 'us.anthropic.claude-haiku-4-5-20251001-v1:0';
|
|
193
|
+
/**
|
|
194
|
+
* `onboarded: false` is deliberate and load-bearing. That developer authors
|
|
195
|
+
* PRs, so their commits appear in the CI census that forms the Attribution
|
|
196
|
+
* Coverage denominator, but they produce no attribution items. Coverage
|
|
197
|
+
* therefore lands below 100% for the honest reason — someone has not run
|
|
198
|
+
* setup-otel-sync — which is exactly what the KPI exists to reveal. Seeding
|
|
199
|
+
* everyone would render a permanent 100% and teach the opposite lesson.
|
|
200
|
+
*/
|
|
201
|
+
const DEVELOPERS = [
|
|
202
|
+
{ email: 'alex.rivera@example.com', tool: 'claude-code', model: SONNET, aiRate: 0.74, onboarded: true },
|
|
203
|
+
{ email: 'priya.natarajan@example.com', tool: 'kiro', model: HAIKU, aiRate: 0.61, onboarded: true },
|
|
204
|
+
{ email: 'sam.okafor@example.com', tool: 'claude-code', model: SONNET, aiRate: 0.83, onboarded: true },
|
|
205
|
+
{ email: 'wei.zhang@example.com', tool: 'cursor', model: HAIKU, aiRate: 0.42, onboarded: true },
|
|
206
|
+
{ email: 'jordan.blake@example.com', tool: 'none', model: '', aiRate: 0, onboarded: false },
|
|
207
|
+
];
|
|
208
|
+
const SPAN_TTL_DAYS = 90;
|
|
209
|
+
const COMMIT_TTL_DAYS = 365;
|
|
28
210
|
function randomInt(min, max) {
|
|
29
211
|
return Math.floor(Math.random() * (max - min + 1)) + min;
|
|
30
212
|
}
|
|
31
213
|
function formatDate(date) {
|
|
32
214
|
return date.toISOString().split('T')[0];
|
|
33
215
|
}
|
|
216
|
+
function hex(bytes) {
|
|
217
|
+
return randomBytes(bytes).toString('hex');
|
|
218
|
+
}
|
|
34
219
|
export default {
|
|
35
|
-
description: 'Generate sample
|
|
220
|
+
description: 'Generate sample data for the PRISM dashboards (8 days of events + attribution)',
|
|
36
221
|
options: [
|
|
37
222
|
{ flags: '--region <region>', description: 'AWS region', default: 'us-west-2' },
|
|
38
223
|
{ flags: '--bus <name>', description: 'EventBridge bus name', default: 'prism-d1-metrics' },
|
|
39
224
|
{ flags: '--team <id>', description: 'Team identifier', default: 'demo-team' },
|
|
40
|
-
{ flags: '--repo <name>', description: '
|
|
225
|
+
{ flags: '--repo <name>', description: 'Primary repository name', default: 'prism-d1-sample-app' },
|
|
226
|
+
{ flags: '--table <name>', description: 'Attribution (AI usage) table name', default: 'prism-d1-ai-usage' },
|
|
227
|
+
{ flags: '--no-attribution', description: 'Emit events only; leave the attribution store untouched' },
|
|
228
|
+
{ flags: '--purge-demo', description: 'Delete previously seeded attribution items and exit' },
|
|
229
|
+
{ flags: '--force', description: 'Seed attribution even when the table already holds real data' },
|
|
41
230
|
],
|
|
42
231
|
action(options) {
|
|
43
|
-
const { region, bus, team, repo } = options;
|
|
44
|
-
|
|
45
|
-
|
|
232
|
+
const { region, bus, team, repo, table } = options;
|
|
233
|
+
const seedAttribution = options.attribution !== false;
|
|
234
|
+
// No shell here, so `command -v aws` is not available — `command` is a
|
|
235
|
+
// shell builtin, not a binary. Probe the binary directly instead.
|
|
236
|
+
if (!run('aws', ['--version']).ok) {
|
|
46
237
|
console.error('Error: AWS CLI not found. Install from https://aws.amazon.com/cli/');
|
|
47
238
|
process.exit(1);
|
|
48
239
|
}
|
|
49
|
-
if (
|
|
50
|
-
console.
|
|
51
|
-
|
|
240
|
+
if (options.purgeDemo) {
|
|
241
|
+
console.log(`Purging seeded attribution items from ${table} (${region})...`);
|
|
242
|
+
const removed = purgeDemoAttribution(region, table);
|
|
243
|
+
console.log(`\nRemoved ${removed} seeded item(s). Real attribution data was left untouched.`);
|
|
244
|
+
return;
|
|
52
245
|
}
|
|
53
246
|
console.log('=== PRISM Demo Data Generator ===');
|
|
54
247
|
console.log(`Region: ${region} | Bus: ${bus} | Team: ${team}`);
|
|
248
|
+
console.log(`Attribution: ${seedAttribution ? `${table}` : 'skipped (--no-attribution)'}`);
|
|
55
249
|
console.log('');
|
|
56
|
-
|
|
57
|
-
|
|
250
|
+
if (seedAttribution) {
|
|
251
|
+
const problem = preflightAttributionTable(region, table);
|
|
252
|
+
if (problem) {
|
|
253
|
+
console.error(`Error: ${problem}`);
|
|
254
|
+
console.error('');
|
|
255
|
+
console.error('Attribution seeding writes directly to DynamoDB, which needs permissions the');
|
|
256
|
+
console.error('rest of this command does not: dynamodb:PutItem, BatchWriteItem and DeleteItem');
|
|
257
|
+
console.error(`on ${table}, plus kms:GenerateDataKey and kms:Decrypt on the table's key.`);
|
|
258
|
+
console.error('Only the otel-receiver and attribution-metrics-publisher Lambdas are granted');
|
|
259
|
+
console.error('these by the CDK stack, so a workshop or devbox role will not have them by');
|
|
260
|
+
console.error('default.');
|
|
261
|
+
console.error('');
|
|
262
|
+
console.error(' --no-attribution emit events only; the event-driven widgets still populate');
|
|
263
|
+
console.error('');
|
|
264
|
+
console.error('Attribution-sourced widgets (AI share, coverage, Repository Breakdown, the');
|
|
265
|
+
console.error('observed PRISM level, Developer Productivity) need either those permissions or');
|
|
266
|
+
console.error('a real codeburn sync from Module 04.');
|
|
267
|
+
process.exit(1);
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
if (seedAttribution && !options.force && hasRealAttribution(region, table)) {
|
|
271
|
+
console.error(`Error: ${table} already contains real attribution data.`);
|
|
272
|
+
console.error('');
|
|
273
|
+
console.error('Seeding fabricated commits alongside real developer activity corrupts the');
|
|
274
|
+
console.error('very KPIs this dashboard exists to report — AI share, coverage, $/shipped.');
|
|
275
|
+
console.error('');
|
|
276
|
+
console.error(' --no-attribution emit events only, leave attribution alone');
|
|
277
|
+
console.error(' --force seed anyway (marked demo_seed=true, removable)');
|
|
278
|
+
process.exit(1);
|
|
279
|
+
}
|
|
280
|
+
// A repo set, so the Repository Breakdown view has more than one row. The
|
|
281
|
+
// per-day PR count is distributed across these rather than multiplied by
|
|
282
|
+
// them, so total event volume is unchanged.
|
|
283
|
+
const REPOS = [repo, `${repo}-api`, `${repo}-web`];
|
|
284
|
+
const seedRunId = hex(8);
|
|
58
285
|
let batch = [];
|
|
59
286
|
let total = 0;
|
|
60
287
|
function flush() {
|
|
@@ -80,66 +307,171 @@ export default {
|
|
|
80
307
|
process.stdout.write('.');
|
|
81
308
|
}
|
|
82
309
|
}
|
|
310
|
+
// ---- Attribution accumulators ----
|
|
311
|
+
//
|
|
312
|
+
// Spans and commits are collected SEPARATELY because write order matters.
|
|
313
|
+
// Each COMMIT# put fires a DynamoDB stream event into
|
|
314
|
+
// attribution-metrics-publisher, which resolves origin by querying the
|
|
315
|
+
// user's SPAN# items for a matching trace_id — it ignores the stored
|
|
316
|
+
// ai_origin verdict. If a commit's stream event is processed before its
|
|
317
|
+
// span exists, the join misses and an AI commit publishes as HumanCommits.
|
|
318
|
+
// The receiver avoids this by running processSpans before
|
|
319
|
+
// processAttributionSpans; this mirrors that ordering.
|
|
320
|
+
const spanRequests = [];
|
|
321
|
+
const commitRequests = [];
|
|
322
|
+
/** (user, day, tool, model) → running totals for the OTEL#DAY rollup. */
|
|
323
|
+
const dayRollups = new Map();
|
|
324
|
+
const commitTtl = Math.floor(Date.now() / 1000) + COMMIT_TTL_DAYS * 86400;
|
|
325
|
+
const spanTtl = Math.floor(Date.now() / 1000) + SPAN_TTL_DAYS * 86400;
|
|
326
|
+
/**
|
|
327
|
+
* Seed one commit's attribution facts.
|
|
328
|
+
*
|
|
329
|
+
* `ai_origin` is written explicitly here because the receiver freezes it at
|
|
330
|
+
* ingest and every read path prefers the stored verdict. The SPAN# item is
|
|
331
|
+
* NOT optional for AI commits: attribution-metrics-publisher ignores the
|
|
332
|
+
* stored verdict and re-derives origin by joining traceId against spans, so
|
|
333
|
+
* an AI commit with no matching span publishes as HumanCommits. Omitting
|
|
334
|
+
* spans would make the native CloudWatch graphs contradict the custom
|
|
335
|
+
* widgets on the same dashboard.
|
|
336
|
+
*/
|
|
337
|
+
function seedCommit(dev, repoName, sha, timestamp, isAi, inMain, wasReverted) {
|
|
338
|
+
const traceId = hex(16);
|
|
339
|
+
const sessionId = `sess-${seedRunId}-${hex(4)}`;
|
|
340
|
+
commitRequests.push({
|
|
341
|
+
PutRequest: {
|
|
342
|
+
Item: {
|
|
343
|
+
pk: { S: `REPO#${repoName}` },
|
|
344
|
+
sk: { S: `COMMIT#${sha}` },
|
|
345
|
+
record_type: { S: 'OTEL_ATTR_COMMIT' },
|
|
346
|
+
session_id: { S: sessionId },
|
|
347
|
+
trace_id: { S: traceId },
|
|
348
|
+
project: { S: repoName },
|
|
349
|
+
user: { S: dev.email },
|
|
350
|
+
device_id: { S: `demo-${seedRunId}` },
|
|
351
|
+
in_main: { BOOL: inMain },
|
|
352
|
+
was_reverted: { BOOL: wasReverted },
|
|
353
|
+
timestamp: { S: timestamp },
|
|
354
|
+
updated_at: { S: new Date().toISOString() },
|
|
355
|
+
ttl: { N: String(commitTtl) },
|
|
356
|
+
ai_origin: { S: isAi ? 'ai-generated' : 'human' },
|
|
357
|
+
ai_tool: { S: isAi ? dev.tool : 'none' },
|
|
358
|
+
ai_model: { S: isAi ? dev.model : '' },
|
|
359
|
+
origin_source: { S: 'write-time-join' },
|
|
360
|
+
// Sparse by-user GSI keys — the per-developer productivity query
|
|
361
|
+
// reads gsi_user / gsi_user_sk rather than scanning.
|
|
362
|
+
gsi_user: { S: `USER#${dev.email}` },
|
|
363
|
+
gsi_user_sk: { S: `COMMIT#${timestamp}` },
|
|
364
|
+
demo_seed: { BOOL: true },
|
|
365
|
+
seed_run_id: { S: seedRunId },
|
|
366
|
+
},
|
|
367
|
+
},
|
|
368
|
+
});
|
|
369
|
+
if (!isAi)
|
|
370
|
+
return;
|
|
371
|
+
const inputTokens = randomInt(3000, 18000);
|
|
372
|
+
const outputTokens = randomInt(800, 6000);
|
|
373
|
+
const costUsd = Math.round((inputTokens / 1000 * 0.003 + outputTokens / 1000 * 0.015) * 10000) / 10000;
|
|
374
|
+
spanRequests.push({
|
|
375
|
+
PutRequest: {
|
|
376
|
+
Item: {
|
|
377
|
+
pk: { S: `USER#${dev.email}` },
|
|
378
|
+
sk: { S: `SPAN#${timestamp}#${hex(8)}` },
|
|
379
|
+
record_type: { S: 'OTEL_SPAN' },
|
|
380
|
+
trace_id: { S: traceId },
|
|
381
|
+
tool: { S: dev.tool },
|
|
382
|
+
model: { S: dev.model },
|
|
383
|
+
input_tokens: { N: String(inputTokens) },
|
|
384
|
+
output_tokens: { N: String(outputTokens) },
|
|
385
|
+
cost_usd: { N: String(costUsd) },
|
|
386
|
+
project: { S: repoName },
|
|
387
|
+
device_id: { S: `demo-${seedRunId}` },
|
|
388
|
+
cost_estimated: { BOOL: false },
|
|
389
|
+
timestamp: { S: timestamp },
|
|
390
|
+
ttl: { N: String(spanTtl) },
|
|
391
|
+
demo_seed: { BOOL: true },
|
|
392
|
+
seed_run_id: { S: seedRunId },
|
|
393
|
+
},
|
|
394
|
+
},
|
|
395
|
+
});
|
|
396
|
+
const day = timestamp.slice(0, 10);
|
|
397
|
+
const key = `${dev.email}|${day}|${dev.tool}|${dev.model}`;
|
|
398
|
+
const roll = dayRollups.get(key) ?? {
|
|
399
|
+
user: dev.email, day, tool: dev.tool, model: dev.model,
|
|
400
|
+
inputTokens: 0, outputTokens: 0, costUsd: 0, calls: 0,
|
|
401
|
+
};
|
|
402
|
+
roll.inputTokens += inputTokens;
|
|
403
|
+
roll.outputTokens += outputTokens;
|
|
404
|
+
roll.costUsd = Math.round((roll.costUsd + costUsd) * 10000) / 10000;
|
|
405
|
+
roll.calls += 1;
|
|
406
|
+
dayRollups.set(key, roll);
|
|
407
|
+
}
|
|
83
408
|
for (let day = 7; day >= 0; day--) {
|
|
84
409
|
const date = new Date();
|
|
85
410
|
date.setDate(date.getDate() - day);
|
|
86
411
|
const dateStr = formatDate(date);
|
|
87
412
|
const maxH = day === 0 ? Math.max(0, new Date().getUTCHours() - 1) : 17;
|
|
88
413
|
const minH = Math.min(8, maxH);
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
for (let i = 0; i < commits; i++) {
|
|
92
|
-
const idx = randomInt(0, 4);
|
|
93
|
-
const origin = ORIGINS[idx];
|
|
94
|
-
const tool = TOOLS[idx];
|
|
95
|
-
const h = randomInt(minH, maxH);
|
|
96
|
-
const m = randomInt(0, 59);
|
|
97
|
-
const ts = `${dateStr}T${String(h).padStart(2, '0')}:${String(m).padStart(2, '0')}:00Z`;
|
|
98
|
-
const lines = randomInt(10, 129);
|
|
99
|
-
addEvent('prism.d1.commit', {
|
|
100
|
-
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
101
|
-
metric: { name: 'commit', value: 1, unit: 'count' },
|
|
102
|
-
ai_context: { tool, model: 'us.anthropic.claude-haiku-4-5-20251001-v1:0', origin },
|
|
103
|
-
ai_dora: { ai_to_merge_ratio: origin !== 'human' ? 1 : 0 },
|
|
104
|
-
});
|
|
105
|
-
}
|
|
106
|
-
// 2-4 PRs per day
|
|
414
|
+
const at = (h, m) => `${dateStr}T${String(h).padStart(2, '0')}:${String(m).padStart(2, '0')}:00Z`;
|
|
415
|
+
// ---- Merged PRs: the census half of Attribution Coverage ----
|
|
107
416
|
const prs = randomInt(2, 4);
|
|
108
417
|
for (let i = 0; i < prs; i++) {
|
|
418
|
+
const prRepo = REPOS[randomInt(0, REPOS.length - 1)];
|
|
419
|
+
const dev = DEVELOPERS[randomInt(0, DEVELOPERS.length - 1)];
|
|
109
420
|
const tc = randomInt(2, 6);
|
|
110
|
-
const
|
|
111
|
-
const
|
|
421
|
+
const shas = Array.from({ length: tc }, () => hex(20));
|
|
422
|
+
const isFailureFix = randomInt(0, 7) === 0;
|
|
112
423
|
const lt = 3600 * (day + 1) + randomInt(0, 3599);
|
|
113
|
-
const
|
|
114
|
-
const
|
|
115
|
-
|
|
116
|
-
//
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
424
|
+
const ts = at(randomInt(minH, maxH), 30);
|
|
425
|
+
const prNumber = 100 + day * 10 + i;
|
|
426
|
+
// Shape mirrors bootstrapper/github-workflows/prism-ai-metrics.yml.
|
|
427
|
+
// Three fields were previously missing or misplaced, each with a
|
|
428
|
+
// visible consequence: pr.total_commits (coverage read "no CI commit
|
|
429
|
+
// census"), pr.commit_shas (security findings had nothing to join for
|
|
430
|
+
// origin), and dora.is_failure_fix (Revert Rate read 0% and Revert
|
|
431
|
+
// Turnaround read "—"). dora.deployment_frequency is deliberately NOT
|
|
432
|
+
// here: it lives only on the deploy event, and carrying it on both
|
|
433
|
+
// double-counted DeploymentFrequency, inflating Merge Frequency 2x.
|
|
120
434
|
addEvent('prism.d1.pr', {
|
|
121
|
-
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
122
|
-
metric: { name: '
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
435
|
+
team_id: team, repo: prRepo, timestamp: ts, prism_level: 2,
|
|
436
|
+
metric: { name: 'pr_merged', value: 1, unit: 'count' },
|
|
437
|
+
dora: { lead_time_seconds: lt, is_failure_fix: isFailureFix },
|
|
438
|
+
pr: {
|
|
439
|
+
number: prNumber,
|
|
440
|
+
author: dev.email.split('@')[0],
|
|
441
|
+
reviews_approved: randomInt(1, 2),
|
|
442
|
+
reviews_changes_requested: randomInt(0, 1),
|
|
127
443
|
total_commits: tc,
|
|
128
|
-
|
|
129
|
-
total_input_tokens: inputTokens,
|
|
130
|
-
total_output_tokens: outputTokens,
|
|
131
|
-
total_cost_usd: costUsd,
|
|
444
|
+
commit_shas: shas,
|
|
132
445
|
},
|
|
133
|
-
pr: { number: pr, author: 'engineer' },
|
|
134
446
|
});
|
|
135
|
-
// Deploy event
|
|
136
447
|
addEvent('prism.d1.deploy', {
|
|
137
|
-
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
448
|
+
team_id: team, repo: prRepo, timestamp: ts, prism_level: 2,
|
|
138
449
|
metric: { name: 'deployment', value: 1, unit: 'count' },
|
|
139
450
|
dora: { deployment_frequency: 1 },
|
|
140
451
|
});
|
|
452
|
+
// Commit events and attribution facts describe the SAME commits, so the
|
|
453
|
+
// two stores agree. Emitting them independently is how the previous
|
|
454
|
+
// version drifted.
|
|
455
|
+
for (const sha of shas) {
|
|
456
|
+
const cts = at(randomInt(minH, maxH), randomInt(0, 59));
|
|
457
|
+
const isAi = dev.onboarded && Math.random() < dev.aiRate;
|
|
458
|
+
const inMain = randomInt(0, 19) !== 0; // ~95% merged
|
|
459
|
+
const wasReverted = inMain && randomInt(0, 24) === 0; // ~4% of merged
|
|
460
|
+
addEvent('prism.d1.commit', {
|
|
461
|
+
team_id: team, repo: prRepo, timestamp: cts, prism_level: 2,
|
|
462
|
+
metric: { name: 'commit', value: 1, unit: 'count' },
|
|
463
|
+
ai_context: {
|
|
464
|
+
tool: isAi ? dev.tool : 'n/a',
|
|
465
|
+
model: isAi ? dev.model : 'n/a',
|
|
466
|
+
origin: isAi ? 'ai-generated' : 'human',
|
|
467
|
+
},
|
|
468
|
+
});
|
|
469
|
+
if (seedAttribution && dev.onboarded) {
|
|
470
|
+
seedCommit(dev, prRepo, sha, cts, isAi, inMain, wasReverted);
|
|
471
|
+
}
|
|
472
|
+
}
|
|
141
473
|
}
|
|
142
|
-
// Eval results
|
|
474
|
+
// ---- Eval gate results ----
|
|
143
475
|
for (let i = 0; i < prs; i++) {
|
|
144
476
|
let score = 0.88;
|
|
145
477
|
let res = 'PASS';
|
|
@@ -151,23 +483,24 @@ export default {
|
|
|
151
483
|
score = 0.71;
|
|
152
484
|
res = 'FAIL';
|
|
153
485
|
}
|
|
154
|
-
const ts = `${dateStr}T${String(randomInt(minH, maxH)).padStart(2, '0')}:45:00Z`;
|
|
155
486
|
addEvent('prism.d1.eval', {
|
|
156
|
-
team_id: team, repo, timestamp:
|
|
487
|
+
team_id: team, repo, timestamp: at(randomInt(minH, maxH), 45), prism_level: 2,
|
|
157
488
|
metric: { name: 'eval_score', value: score, unit: 'score' },
|
|
158
|
-
ai_context: { tool: 'bedrock-eval', model:
|
|
489
|
+
ai_context: { tool: 'bedrock-eval', model: HAIKU, origin: 'ai-generated' },
|
|
159
490
|
ai_dora: { eval_gate_pass_rate: res === 'PASS' ? 1 : 0 },
|
|
160
491
|
eval: { result: res, pr_number: 100 + day * 10 + i },
|
|
161
492
|
});
|
|
162
493
|
}
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
494
|
+
const ts = at(9, 0);
|
|
495
|
+
// Change failure rate and MTTR still have live CloudWatch metrics and an
|
|
496
|
+
// alarm. The other assessment metrics this loop used to emit —
|
|
497
|
+
// ai_acceptance_rate, ai_test_coverage_delta, spec_to_code_hours,
|
|
498
|
+
// post_merge_defect_rate, PRISMLevel, BedrockCostUSD, TokenEfficiency —
|
|
499
|
+
// have no consumer left: their metrics were deleted, their producing
|
|
500
|
+
// Lambdas were removed, or the exec view now computes the value rather
|
|
501
|
+
// than reading a published one.
|
|
502
|
+
const cfr = parseFloat((randomInt(1, 8) / 100).toFixed(4));
|
|
503
|
+
const mttrSeconds = randomInt(600, 4199);
|
|
171
504
|
addEvent('prism.d1.assessment', {
|
|
172
505
|
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
173
506
|
metric: { name: 'change_failure_rate', value: cfr, unit: 'percent' },
|
|
@@ -175,122 +508,73 @@ export default {
|
|
|
175
508
|
});
|
|
176
509
|
addEvent('prism.d1.assessment', {
|
|
177
510
|
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
178
|
-
metric: { name: 'mttr', value:
|
|
179
|
-
dora: { mttr_seconds:
|
|
180
|
-
});
|
|
181
|
-
addEvent('prism.d1.assessment', {
|
|
182
|
-
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
183
|
-
metric: { name: 'ai_acceptance_rate', value: accept, unit: 'percent' },
|
|
184
|
-
ai_dora: { ai_acceptance_rate: accept },
|
|
185
|
-
});
|
|
186
|
-
addEvent('prism.d1.assessment', {
|
|
187
|
-
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
188
|
-
metric: { name: 'ai_test_coverage_delta', value: coverage, unit: 'percent' },
|
|
189
|
-
ai_dora: { ai_test_coverage_delta: coverage },
|
|
190
|
-
});
|
|
191
|
-
addEvent('prism.d1.assessment', {
|
|
192
|
-
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
193
|
-
metric: { name: 'spec_to_code_hours', value: specH, unit: 'count' },
|
|
194
|
-
ai_dora: { spec_to_code_hours: specH },
|
|
195
|
-
});
|
|
196
|
-
addEvent('prism.d1.assessment', {
|
|
197
|
-
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
198
|
-
metric: { name: 'post_merge_defect_rate', value: defect, unit: 'percent' },
|
|
199
|
-
ai_dora: { post_merge_defect_rate: defect },
|
|
200
|
-
});
|
|
201
|
-
// PRISM Level (trending up over the week)
|
|
202
|
-
const level = parseFloat((1.5 + (7 - day) * 0.2).toFixed(1));
|
|
203
|
-
addEvent('prism.d1.assessment', {
|
|
204
|
-
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
205
|
-
metric: { name: 'PRISMLevel', value: level, unit: 'none' },
|
|
511
|
+
metric: { name: 'mttr', value: mttrSeconds, unit: 'seconds' },
|
|
512
|
+
dora: { mttr_seconds: mttrSeconds },
|
|
206
513
|
});
|
|
207
|
-
// Agent invocations
|
|
514
|
+
// ---- Agent invocations ----
|
|
208
515
|
const agentCount = randomInt(3, 10);
|
|
209
516
|
for (let a = 0; a < agentCount; a++) {
|
|
210
517
|
const steps = randomInt(2, 9);
|
|
211
|
-
const dur = randomInt(1000, 5999);
|
|
212
|
-
const tokens = randomInt(2000, 9999);
|
|
213
|
-
const status = randomInt(0, 5) === 0 ? 'failure' : 'success';
|
|
214
|
-
const ats = `${dateStr}T${String(randomInt(minH, maxH)).padStart(2, '0')}:${String(randomInt(0, 59)).padStart(2, '0')}:00Z`;
|
|
215
518
|
addEvent('prism.d1.agent', {
|
|
216
|
-
team_id: team, repo, timestamp:
|
|
519
|
+
team_id: team, repo, timestamp: at(randomInt(minH, maxH), randomInt(0, 59)), prism_level: 3,
|
|
217
520
|
metric: { name: 'agent_invocation', value: 1, unit: 'count' },
|
|
218
|
-
ai_context: { tool: 'strands-agent', model:
|
|
521
|
+
ai_context: { tool: 'strands-agent', model: HAIKU, origin: 'ai-generated' },
|
|
219
522
|
agent: {
|
|
220
523
|
agent_name: 'task-assistant', steps_taken: steps, tools_invoked: steps - 1,
|
|
221
|
-
duration_ms:
|
|
524
|
+
duration_ms: randomInt(1000, 5999), tokens_used: randomInt(2000, 9999),
|
|
525
|
+
status: randomInt(0, 5) === 0 ? 'failure' : 'success', guardrails_triggered: 0,
|
|
222
526
|
},
|
|
223
527
|
});
|
|
224
528
|
}
|
|
225
|
-
// Guardrail triggers
|
|
529
|
+
// ---- Guardrail triggers ----
|
|
226
530
|
const GUARDRAIL_CATEGORIES = ['CONTENT_FILTER', 'DENIED_TOPIC', 'SENSITIVE_INFO', 'WORD_FILTER'];
|
|
227
531
|
const GUARDRAIL_ACTIONS = ['BLOCK', 'ANONYMIZE', 'WARN'];
|
|
228
|
-
|
|
229
|
-
for (let g = 0; g < guardrailCount; g++) {
|
|
230
|
-
const gts = `${dateStr}T${String(randomInt(minH, maxH)).padStart(2, '0')}:${String(randomInt(0, 59)).padStart(2, '0')}:00Z`;
|
|
231
|
-
const category = GUARDRAIL_CATEGORIES[randomInt(0, 3)];
|
|
232
|
-
const action = GUARDRAIL_ACTIONS[randomInt(0, 2)];
|
|
532
|
+
for (let g = 0; g < randomInt(1, 5); g++) {
|
|
233
533
|
addEvent('prism.d1.guardrail', {
|
|
234
|
-
team_id: team, repo, timestamp:
|
|
534
|
+
team_id: team, repo, timestamp: at(randomInt(minH, maxH), randomInt(0, 59)), prism_level: 2,
|
|
235
535
|
metric: { name: 'guardrail_trigger', value: 1, unit: 'count' },
|
|
236
536
|
guardrail: {
|
|
237
537
|
guardrail_id: 'gr-demo-001', guardrail_name: 'prism-safety',
|
|
238
|
-
trigger_category:
|
|
239
|
-
action_taken:
|
|
538
|
+
trigger_category: GUARDRAIL_CATEGORIES[randomInt(0, 3)], trigger_type: 'automated',
|
|
539
|
+
action_taken: GUARDRAIL_ACTIONS[randomInt(0, 2)], agent_name: 'task-assistant',
|
|
540
|
+
invocation_id: `inv-${day}-${g}`,
|
|
240
541
|
},
|
|
241
542
|
});
|
|
242
543
|
}
|
|
243
|
-
// MCP tool calls
|
|
544
|
+
// ---- MCP tool calls ----
|
|
244
545
|
const MCP_TOOLS = ['file_read', 'file_write', 'shell_exec', 'web_fetch', 'db_query'];
|
|
245
|
-
|
|
246
|
-
for (let m = 0; m < mcpCount; m++) {
|
|
247
|
-
const mts = `${dateStr}T${String(randomInt(minH, maxH)).padStart(2, '0')}:${String(randomInt(0, 59)).padStart(2, '0')}:00Z`;
|
|
546
|
+
for (let m = 0; m < randomInt(5, 15); m++) {
|
|
248
547
|
const authorized = randomInt(0, 19) !== 0; // 5% denied
|
|
249
548
|
addEvent('prism.d1.mcp.tool_call', {
|
|
250
|
-
team_id: team, repo, timestamp:
|
|
549
|
+
team_id: team, repo, timestamp: at(randomInt(minH, maxH), randomInt(0, 59)), prism_level: 2,
|
|
251
550
|
metric: { name: 'mcp_tool_call', value: 1, unit: 'count' },
|
|
252
551
|
mcp_tool_call: {
|
|
253
552
|
session_id: `sess-${day}-${m}`, client_id: 'claude-code',
|
|
254
|
-
tool_name: MCP_TOOLS[randomInt(0, 4)],
|
|
255
|
-
scopes_used: ['read'], authorized,
|
|
553
|
+
tool_name: MCP_TOOLS[randomInt(0, 4)], scopes_used: ['read'], authorized,
|
|
256
554
|
risk_level: authorized ? 'low' : 'high',
|
|
257
555
|
duration_ms: randomInt(50, 2000),
|
|
258
556
|
result_status: authorized ? 'success' : 'denied',
|
|
259
557
|
},
|
|
260
558
|
});
|
|
261
559
|
}
|
|
262
|
-
//
|
|
263
|
-
const dailyCost = parseFloat((randomInt(15, 45) + randomInt(0, 99) / 100).toFixed(2));
|
|
264
|
-
const tokenEff = randomInt(80, 300);
|
|
265
|
-
addEvent('prism.d1.assessment', {
|
|
266
|
-
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
267
|
-
metric: { name: 'BedrockCostUSD', value: dailyCost, unit: 'none' },
|
|
268
|
-
});
|
|
269
|
-
addEvent('prism.d1.assessment', {
|
|
270
|
-
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
271
|
-
metric: { name: 'TokenEfficiency', value: tokenEff, unit: 'none' },
|
|
272
|
-
});
|
|
273
|
-
// Quality: AI vs Human defect rates (Pillar 7)
|
|
274
|
-
const aiDefect = parseFloat((randomInt(1, 4) / 100).toFixed(4));
|
|
275
|
-
const humanDefect = parseFloat((randomInt(2, 7) / 100).toFixed(4));
|
|
560
|
+
// ---- Quality: AI vs human defect rates ----
|
|
276
561
|
addEvent('prism.d1.quality', {
|
|
277
562
|
team_id: team, repo, timestamp: ts, prism_level: 2,
|
|
278
563
|
metric: { name: 'quality_comparison', value: 1, unit: 'count' },
|
|
279
564
|
quality: {
|
|
280
565
|
deployment_id: `deploy-${day}`,
|
|
281
|
-
ai_defect_rate:
|
|
566
|
+
ai_defect_rate: parseFloat((randomInt(1, 4) / 100).toFixed(4)),
|
|
567
|
+
human_defect_rate: parseFloat((randomInt(2, 7) / 100).toFixed(4)),
|
|
282
568
|
total_ai_commits: randomInt(5, 12), total_human_commits: randomInt(3, 8),
|
|
283
569
|
},
|
|
284
570
|
});
|
|
285
|
-
// Security
|
|
571
|
+
// ---- Security findings ----
|
|
286
572
|
const PHASES = ['design_review', 'code_review', 'pen_test'];
|
|
287
573
|
const SEVERITIES = ['CRITICAL', 'HIGH', 'MEDIUM', 'LOW'];
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
const fts = `${dateStr}T${String(randomInt(minH, maxH)).padStart(2, '0')}:${String(randomInt(0, 59)).padStart(2, '0')}:00Z`;
|
|
574
|
+
for (let f = 0; f < randomInt(1, 4); f++) {
|
|
575
|
+
const fts = at(randomInt(minH, maxH), randomInt(0, 59));
|
|
291
576
|
const severity = SEVERITIES[randomInt(0, 3)];
|
|
292
577
|
const phase = PHASES[randomInt(0, 2)];
|
|
293
|
-
const aiOrigin = randomInt(0, 1) === 0 ? 'ai-assisted' : 'human';
|
|
294
578
|
addEvent('prism.d1.security.code_review', {
|
|
295
579
|
team_id: team, repo, timestamp: fts, prism_level: 2,
|
|
296
580
|
metric: { name: 'security_finding', value: 1, unit: 'count' },
|
|
@@ -299,60 +583,112 @@ export default {
|
|
|
299
583
|
cvss_score: severity === 'CRITICAL' ? 9.1 : severity === 'HIGH' ? 7.5 : severity === 'MEDIUM' ? 5.2 : 2.8,
|
|
300
584
|
title: 'Demo finding', category: 'injection',
|
|
301
585
|
cwe_id: 'CWE-79', exploit_validated: phase === 'pen_test',
|
|
302
|
-
compliance_mappings: ['OWASP-A03'],
|
|
586
|
+
compliance_mappings: ['OWASP-A03'],
|
|
587
|
+
ai_origin: randomInt(0, 1) === 0 ? 'ai-assisted' : 'human',
|
|
303
588
|
spec_ref: null, found_at: fts, remediated_at: null,
|
|
304
589
|
},
|
|
305
590
|
});
|
|
306
591
|
}
|
|
307
|
-
// Security remediation
|
|
592
|
+
// ---- Security remediation ----
|
|
308
593
|
if (day > 0 && randomInt(0, 2) === 0) {
|
|
309
|
-
const rts = `${dateStr}T${String(randomInt(minH, maxH)).padStart(2, '0')}:${String(randomInt(0, 59)).padStart(2, '0')}:00Z`;
|
|
310
|
-
const remOrigin = randomInt(0, 1) === 0 ? 'ai-assisted' : 'human';
|
|
311
594
|
addEvent('prism.d1.security.remediation', {
|
|
312
|
-
team_id: team, repo, timestamp:
|
|
595
|
+
team_id: team, repo, timestamp: at(randomInt(minH, maxH), randomInt(0, 59)), prism_level: 2,
|
|
313
596
|
metric: { name: 'security_remediation', value: 1, unit: 'count' },
|
|
314
597
|
security_remediation: {
|
|
315
598
|
finding_id: `finding-${day + 1}-0`,
|
|
316
599
|
severity: SEVERITIES[randomInt(0, 2)],
|
|
317
600
|
remediation_time_hours: randomInt(2, 48),
|
|
318
|
-
remediated_by_origin:
|
|
601
|
+
remediated_by_origin: randomInt(0, 1) === 0 ? 'ai-assisted' : 'human',
|
|
319
602
|
finding_phase: 'code_review',
|
|
320
603
|
},
|
|
321
604
|
});
|
|
322
605
|
}
|
|
323
|
-
// Exfiltration alert (rare)
|
|
606
|
+
// ---- Exfiltration alert (rare) ----
|
|
324
607
|
if (randomInt(0, 6) === 0) {
|
|
325
|
-
const ets =
|
|
608
|
+
const ets = at(randomInt(minH, maxH), randomInt(0, 59));
|
|
326
609
|
addEvent('prism.d1.security', {
|
|
327
610
|
team_id: team, repo, timestamp: ets, prism_level: 2,
|
|
328
611
|
metric: { name: 'exfiltration_alert', value: 1, unit: 'count' },
|
|
329
612
|
security: {
|
|
330
613
|
alert_type: 'anomalous_read', table_name: 'prism-d1-events',
|
|
331
614
|
principal_arn: 'arn:aws:iam::123456789012:role/demo-role',
|
|
332
|
-
read_count: randomInt(500, 2000),
|
|
333
|
-
window_start: ets, window_end: ets,
|
|
615
|
+
read_count: randomInt(500, 2000), window_start: ets, window_end: ets,
|
|
334
616
|
},
|
|
335
617
|
});
|
|
336
618
|
}
|
|
337
|
-
//
|
|
619
|
+
// ---- Per-rubric eval scores ----
|
|
338
620
|
const RUBRICS = ['code-quality', 'api-response-quality', 'agent-quality', 'security-compliance', 'spec-compliance'];
|
|
339
621
|
for (const rubric of RUBRICS) {
|
|
340
622
|
const rScore = parseFloat((randomInt(65, 98) / 100).toFixed(2));
|
|
341
|
-
const rts = `${dateStr}T${String(randomInt(minH, maxH)).padStart(2, '0')}:50:00Z`;
|
|
342
623
|
addEvent('prism.d1.eval', {
|
|
343
|
-
team_id: team, repo, timestamp:
|
|
624
|
+
team_id: team, repo, timestamp: at(randomInt(minH, maxH), 50), prism_level: 2,
|
|
344
625
|
metric: { name: 'eval_score', value: rScore, unit: 'score' },
|
|
345
|
-
ai_context: { tool: 'bedrock-eval', model:
|
|
626
|
+
ai_context: { tool: 'bedrock-eval', model: HAIKU, origin: 'ai-generated' },
|
|
346
627
|
ai_dora: { eval_gate_pass_rate: rScore >= 0.7 ? 1 : 0 },
|
|
347
628
|
eval: { result: rScore >= 0.7 ? 'PASS' : 'FAIL', rubric, score: rScore, pr_number: 100 + day * 10 },
|
|
348
629
|
});
|
|
349
630
|
}
|
|
350
631
|
}
|
|
351
632
|
flush();
|
|
352
|
-
console.log('
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
633
|
+
console.log('');
|
|
634
|
+
// ---- Attribution store ----
|
|
635
|
+
let attrWritten = 0;
|
|
636
|
+
if (seedAttribution) {
|
|
637
|
+
for (const roll of dayRollups.values()) {
|
|
638
|
+
spanRequests.push({
|
|
639
|
+
PutRequest: {
|
|
640
|
+
Item: {
|
|
641
|
+
pk: { S: `USER#${roll.user}` },
|
|
642
|
+
// '#' is the key delimiter, so it must not appear in the model.
|
|
643
|
+
sk: { S: `OTEL#DAY#${roll.day}#${roll.tool}#${roll.model.replace(/#/g, '')}` },
|
|
644
|
+
record_type: { S: 'OTEL_DAY' },
|
|
645
|
+
tool: { S: roll.tool },
|
|
646
|
+
model: { S: roll.model.replace(/#/g, '') },
|
|
647
|
+
day: { S: roll.day },
|
|
648
|
+
input_tokens: { N: String(roll.inputTokens) },
|
|
649
|
+
output_tokens: { N: String(roll.outputTokens) },
|
|
650
|
+
cost_usd: { N: String(roll.costUsd) },
|
|
651
|
+
call_count: { N: String(roll.calls) },
|
|
652
|
+
updated_at: { S: new Date().toISOString() },
|
|
653
|
+
demo_seed: { BOOL: true },
|
|
654
|
+
seed_run_id: { S: seedRunId },
|
|
655
|
+
},
|
|
656
|
+
},
|
|
657
|
+
});
|
|
658
|
+
}
|
|
659
|
+
const totalItems = spanRequests.length + commitRequests.length;
|
|
660
|
+
if (totalItems > 0) {
|
|
661
|
+
console.log(`Seeding ${totalItems} attribution items into ${table}...`);
|
|
662
|
+
// Spans and rollups first — see the note on the accumulators above.
|
|
663
|
+
const spans = ddbBatchWrite(region, table, spanRequests);
|
|
664
|
+
attrWritten += spans.written;
|
|
665
|
+
let failure = spans.error;
|
|
666
|
+
if (!failure) {
|
|
667
|
+
const commits = ddbBatchWrite(region, table, commitRequests);
|
|
668
|
+
attrWritten += commits.written;
|
|
669
|
+
failure = commits.error;
|
|
670
|
+
}
|
|
671
|
+
console.log('');
|
|
672
|
+
if (failure) {
|
|
673
|
+
console.error(`Error: attribution seeding stopped after ${attrWritten} of ${totalItems} items.`);
|
|
674
|
+
console.error(failure);
|
|
675
|
+
console.error('');
|
|
676
|
+
console.error('Events were emitted successfully, so the event-driven widgets will populate.');
|
|
677
|
+
console.error(`Clean up the partial seed with: prism-cli workshop generate-demo-data --purge-demo --region ${region}`);
|
|
678
|
+
process.exit(1);
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
console.log(`=== Done! ${total} events emitted, ${attrWritten} attribution items seeded ===`);
|
|
683
|
+
if (seedAttribution) {
|
|
684
|
+
const onboarded = DEVELOPERS.filter(d => d.onboarded).length;
|
|
685
|
+
console.log(`Fleet: ${onboarded} of ${DEVELOPERS.length} developers onboarded — Attribution Coverage`);
|
|
686
|
+
console.log('will read below 100% by design, which is what the KPI is for.');
|
|
687
|
+
console.log(`Seed run ${seedRunId} · remove with: prism-cli workshop generate-demo-data --purge-demo`);
|
|
688
|
+
}
|
|
689
|
+
console.log('');
|
|
690
|
+
console.log(`Open CloudWatch → Dashboards → PRISM-D1-Team-Velocity (${region})`);
|
|
691
|
+
console.log("Set the time range to 'Last 1 week'.");
|
|
356
692
|
},
|
|
357
693
|
};
|
|
358
694
|
//# sourceMappingURL=generate-demo-data.js.map
|